From fa574cfbec519c2eba6fbdbf7d414e77b65ff2fd Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Fri, 6 Mar 2020 15:02:03 -0600 Subject: [PATCH 01/32] allow dfhack-run to output colors. refactor Console-posix to avoid having a parameter equivalent to not calling the function at all. --- library/CMakeLists.txt | 16 +++++++++++----- library/Console-posix.cpp | 24 ++++++++++++++---------- library/Core.cpp | 1 - library/dfhack-run.cpp | 9 +++++++-- library/include/Console.h | 2 +- 5 files changed, 33 insertions(+), 19 deletions(-) diff --git a/library/CMakeLists.txt b/library/CMakeLists.txt index 14be2b555..2de293e15 100644 --- a/library/CMakeLists.txt +++ b/library/CMakeLists.txt @@ -79,8 +79,14 @@ set(MAIN_SOURCES RemoteTools.cpp ) +if(WIN32) + set(CONSOLE_SOURCES Console-windows.cpp) +else() + set(CONSOLE_SOURCES Console-posix.cpp) +endif() + set(MAIN_SOURCES_WINDOWS - Console-windows.cpp + ${CONSOLE_SOURCES} Hooks-windows.cpp PlugLoad-windows.cpp Process-windows.cpp @@ -92,21 +98,21 @@ if(WIN32) endif() set(MAIN_SOURCES_LINUX - Console-posix.cpp + ${CONSOLE_SOURCES} Hooks-linux.cpp PlugLoad-posix.cpp Process-linux.cpp ) set(MAIN_SOURCES_DARWIN - Console-posix.cpp + ${CONSOLE_SOURCES} Hooks-darwin.cpp PlugLoad-posix.cpp Process-darwin.cpp ) set(MAIN_SOURCES_LINUX_EGGY - Console-linux.cpp + ${CONSOLE_SOURCES} Hooks-egg.cpp PlugLoad-linux.cpp Process-linux.cpp @@ -359,7 +365,7 @@ add_library(dfhack SHARED ${PROJECT_SOURCES}) add_dependencies(dfhack generate_proto_core) add_dependencies(dfhack generate_headers) -add_library(dfhack-client SHARED RemoteClient.cpp ColorText.cpp MiscUtils.cpp Error.cpp ${PROJECT_PROTO_SRCS}) +add_library(dfhack-client SHARED RemoteClient.cpp ColorText.cpp MiscUtils.cpp Error.cpp ${PROJECT_PROTO_SRCS} ${CONSOLE_SOURCES}) add_dependencies(dfhack-client dfhack) add_executable(dfhack-run dfhack-run.cpp) diff --git a/library/Console-posix.cpp b/library/Console-posix.cpp index 696672861..9633f80e0 100644 --- a/library/Console-posix.cpp +++ b/library/Console-posix.cpp @@ -827,21 +827,23 @@ Console::~Console() delete d; } -bool Console::init(bool sharing) +bool Console::init(bool dont_redirect) { - if(sharing) - { - inited = false; - return false; - } - if (!freopen("stdout.log", "w", stdout)) - ; d = new Private(); // make our own weird streams so our IO isn't redirected - d->dfout_C = fopen("/dev/tty", "w"); + if (dont_redirect) + { + d->dfout_C = fopen("/dev/stdout", "w"); + } + else + { + if (!freopen("stdout.log", "w", stdout)) + ; + d->dfout_C = fopen("/dev/tty", "w"); + } std::cin.tie(this); clear(); - d->supported_terminal = !isUnsupportedTerm() && isatty(STDIN_FILENO); + d->supported_terminal = !isUnsupportedTerm() && isatty(STDIN_FILENO); // init the exit mechanism if (pipe(d->exit_pipe) == -1) ; @@ -858,6 +860,8 @@ bool Console::shutdown(void) return true; lock_guard g(*wlock); close(d->exit_pipe[1]); + if (d->state != Private::con_lineedit) + inited = false; return true; } diff --git a/library/Core.cpp b/library/Core.cpp index 00a8879fc..c0a1155c7 100644 --- a/library/Core.cpp +++ b/library/Core.cpp @@ -1707,7 +1707,6 @@ bool Core::Init() } if (is_text_mode && !is_headless) { - con.init(true); cerr << "Console is not available. Use dfhack-run to send commands.\n"; if (!is_text_mode) { diff --git a/library/dfhack-run.cpp b/library/dfhack-run.cpp index af00e4c04..df10bf148 100644 --- a/library/dfhack-run.cpp +++ b/library/dfhack-run.cpp @@ -45,6 +45,7 @@ POSSIBILITY OF SUCH DAMAGE. #include #include +#include "Console.h" #include "RemoteClient.h" #include @@ -55,11 +56,10 @@ POSSIBILITY OF SUCH DAMAGE. using namespace DFHack; using namespace dfproto; -using std::cout; int main (int argc, char *argv[]) { - color_ostream_wrapper out(cout); + Console out; if (argc <= 1) { @@ -85,12 +85,15 @@ int main (int argc, char *argv[]) if (!client.connect()) return 2; + out.init(true); + command_result rv; if (strcmp(argv[1], "--lua") == 0) { if (argc <= 3) { + out.shutdown(); fprintf(stderr, "Usage: dfhack-run --lua [args...]\n"); return 2; } @@ -99,6 +102,7 @@ int main (int argc, char *argv[]) if (!run_call.bind(&client, "RunLua")) { + out.shutdown(); fprintf(stderr, "No RunLua protocol function found."); return 3; } @@ -130,6 +134,7 @@ int main (int argc, char *argv[]) } out.flush(); + out.shutdown(); if (rv != CR_OK) return 1; diff --git a/library/include/Console.h b/library/include/Console.h index 0517704d4..0882ba449 100644 --- a/library/include/Console.h +++ b/library/include/Console.h @@ -135,7 +135,7 @@ namespace DFHack ///dtor, NOT thread-safe ~Console(); /// initialize the console. NOT thread-safe - bool init( bool sharing ); + bool init( bool dont_redirect ); /// shutdown the console. NOT thread-safe bool shutdown( void ); From 635e709d95d8ba583f251bdea3a1002f019cd178 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Fri, 6 Mar 2020 16:06:27 -0600 Subject: [PATCH 02/32] name ui_look_list union update scripts and structures --- library/modules/Buildings.cpp | 2 +- library/modules/Gui.cpp | 10 +++++----- library/xml | 2 +- plugins/search.cpp | 12 ++++++------ scripts | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/library/modules/Buildings.cpp b/library/modules/Buildings.cpp index 607f63a36..5219395c1 100644 --- a/library/modules/Buildings.cpp +++ b/library/modules/Buildings.cpp @@ -1162,7 +1162,7 @@ bool Buildings::deconstruct(df::building *bld) { auto item = ui_look_list->items[i]; if (item->type == df::ui_look_list::T_items::Building && - item->building == bld) + item->data.Building == bld) { vector_erase_at(ui_look_list->items, i); delete item; diff --git a/library/modules/Gui.cpp b/library/modules/Gui.cpp index b4c4a7693..3ba851438 100644 --- a/library/modules/Gui.cpp +++ b/library/modules/Gui.cpp @@ -1064,12 +1064,12 @@ df::unit *Gui::getAnyUnit(df::viewscreen *top) if (auto item = vector_get(ui_look_list->items, *ui_look_cursor)) { if (item->type == df::ui_look_list::T_items::Unit) - return item->unit; + return item->data.Unit; else if (item->type == df::ui_look_list::T_items::Item) { - if (VIRTUAL_CAST_VAR(corpse, df::item_corpsest, item->item)) + if (VIRTUAL_CAST_VAR(corpse, df::item_corpsest, item->data.Item)) return df::unit::find(corpse->unit_id); // loo(k) at corpse - else if (VIRTUAL_CAST_VAR(corpsepiece, df::item_corpsepiecest, item->item)) + else if (VIRTUAL_CAST_VAR(corpsepiece, df::item_corpsepiecest, item->data.Item)) return df::unit::find(corpsepiece->unit_id); // loo(k) at corpse piece } else if (item->type == df::ui_look_list::T_items::Spatter) @@ -1201,7 +1201,7 @@ df::item *Gui::getAnyItem(df::viewscreen *top) auto item = vector_get(ui_look_list->items, *ui_look_cursor); if (item && item->type == df::ui_look_list::T_items::Item) - return item->item; + return item->data.Item; else return NULL; } @@ -1266,7 +1266,7 @@ df::building *Gui::getAnyBuilding(df::viewscreen *top) auto item = vector_get(ui_look_list->items, *ui_look_cursor); if (item && item->type == df::ui_look_list::T_items::Building) - return item->building; + return item->data.Building; else return NULL; } diff --git a/library/xml b/library/xml index 167d54bf7..23e66dc0d 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit 167d54bf7c0e01a71ffb2de548701adc93da9dad +Subproject commit 23e66dc0d44287a63fd918bbb81681510850c37d diff --git a/plugins/search.cpp b/plugins/search.cpp index 452e8cc23..43e14996a 100644 --- a/plugins/search.cpp +++ b/plugins/search.cpp @@ -1822,16 +1822,16 @@ public: switch (element->type) { case elt_type::Item: - if (element->item) - desc = Items::getDescription(element->item, 0, true); + if (element->data.Item) + desc = Items::getDescription(element->data.Item, 0, true); break; case elt_type::Unit: - if (element->unit) - desc = get_unit_description(element->unit); + if (element->data.Unit) + desc = get_unit_description(element->data.Unit); break; case elt_type::Building: - if (element->building) - element->building->getName(&desc); + if (element->data.Building) + element->data.Building->getName(&desc); break; default: break; diff --git a/scripts b/scripts index 7bab11642..f19655a4f 160000 --- a/scripts +++ b/scripts @@ -1 +1 @@ -Subproject commit 7bab11642bee7a3aa05d69332466f2ea5eaa1a2d +Subproject commit f19655a4fc7cb89391cb484d04ba1dff81a11d6d From 508ab79af267259a48fc21dfa650147b9549e549 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Fri, 6 Mar 2020 16:33:02 -0600 Subject: [PATCH 03/32] document where tagged unions should be implemented for Lua --- library/LuaTypes.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/library/LuaTypes.cpp b/library/LuaTypes.cpp index 3144ff7a6..d289984ff 100644 --- a/library/LuaTypes.cpp +++ b/library/LuaTypes.cpp @@ -1165,6 +1165,7 @@ static void IndexFields(lua_State *state, int base, struct_identity *pstruct, bo lua_pop(state, 1); bool add_to_enum = true; + const struct_field_info *tag_field = nullptr; // Handle the field switch (fields[i].mode) @@ -1183,6 +1184,11 @@ static void IndexFields(lua_State *state, int base, struct_identity *pstruct, bo add_to_enum = false; break; + case struct_field_info::SUBSTRUCT: + case struct_field_info::CONTAINER: + tag_field = find_union_tag(fields, &fields[i]); + break; + default: break; } @@ -1194,6 +1200,17 @@ static void IndexFields(lua_State *state, int base, struct_identity *pstruct, bo if (add_to_enum) AssociateId(state, base+3, ++cnt, name.c_str()); + if (tag_field) + { + // TODO: handle tagged unions + // + // tagged unions are treated as if they have at most one field, + // with the same name as the corresponding enumeration value. + // + // if no field's name matches the enumeration value's name, + // the tagged union is treated as a structure with no fields. + } + lua_pushlightuserdata(state, (void*)&fields[i]); lua_setfield(state, base+2, name.c_str()); } From a214e004074e74510ad3d0b959b3298b1721a2d3 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Fri, 6 Mar 2020 16:48:33 -0600 Subject: [PATCH 04/32] allow union vectors to have tags that are bit vectors if they have exactly 2 members --- library/DataDefs.cpp | 9 +++++ library/include/DataDefs.h | 3 ++ plugins/devel/check-structures-sanity.cpp | 41 +++++++++++++++++++++-- 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/library/DataDefs.cpp b/library/DataDefs.cpp index cd3474619..c071ad7d3 100644 --- a/library/DataDefs.cpp +++ b/library/DataDefs.cpp @@ -539,5 +539,14 @@ const struct_field_info *DFHack::find_union_tag(const struct_field_info *fields, return tag_candidate; } + auto union_fields = ((struct_identity*)union_field->type)->getFields(); + if (tag_container_type->getFullName(nullptr) != "vector" && + union_fields[0].mode != struct_field_info::END && + union_fields[1].mode != struct_field_info::END && + union_fields[2].mode == struct_field_info::END) + { + return tag_candidate; + } + return nullptr; } diff --git a/library/include/DataDefs.h b/library/include/DataDefs.h index 704b1abf7..03bb7857f 100644 --- a/library/include/DataDefs.h +++ b/library/include/DataDefs.h @@ -804,6 +804,9 @@ namespace DFHack { * * If the union field is a container type, the returned tag field is * a container of primitive enum types. + * + * As a special case, a container-type union can have a tag field that is + * a bit vector if it has exactly two members. */ DFHACK_EXPORT const struct_field_info *find_union_tag(const struct_field_info *fields, const struct_field_info *union_field); } diff --git a/plugins/devel/check-structures-sanity.cpp b/plugins/devel/check-structures-sanity.cpp index b5cd1aa5f..3a454a0e6 100644 --- a/plugins/devel/check-structures-sanity.cpp +++ b/plugins/devel/check-structures-sanity.cpp @@ -150,6 +150,7 @@ private: bool maybe_queue_union(const ToCheck &, const struct_field_info *, const struct_field_info *); void queue_union(const ToCheck &, const ToCheck &); void queue_union_vector(const ToCheck &, const ToCheck &); + void queue_union_bitvector(const ToCheck &, const ToCheck &); void check_dispatch(ToCheck &); void check_global(const ToCheck &); void check_primitive(const ToCheck &); @@ -163,7 +164,7 @@ private: void check_deque(const ToCheck &, type_identity *); void check_dfarray(const ToCheck &, type_identity *); void check_bitarray(const ToCheck &); - void check_bitvector(const ToCheck &); + bool check_bitvector(const ToCheck &); void check_struct(const ToCheck &); void check_virtual(const ToCheck &); public: @@ -691,6 +692,12 @@ void Checker::queue_union(const ToCheck & item, const ToCheck & tag_item) void Checker::queue_union_vector(const ToCheck & item, const ToCheck & tag_item) { + if (tag_item.identity->getFullName(nullptr) == "vector") + { + queue_union_bitvector(item, tag_item); + return; + } + auto union_type = static_cast(static_cast(item.identity)->getItemType()); auto tag_type = static_cast(static_cast(tag_item.identity)->getItemType()); @@ -714,6 +721,33 @@ void Checker::queue_union_vector(const ToCheck & item, const ToCheck & tag_item) } } +void Checker::queue_union_bitvector(const ToCheck & item, const ToCheck & tag_item) +{ + auto union_type = static_cast(static_cast(item.identity)->getItemType()); + auto union_count = check_vector_size(item, union_type->byte_size()); + + if (!check_bitvector(tag_item)) + { + return; + } + auto tag_vector = reinterpret_cast *>(tag_item.ptr); + + if (union_count != tag_vector->size()) + { + FAIL("tagged union vector size (" << union_count << ") does not match tag vector (" << join_strings("", tag_item.path) << ") size (" << tag_count << ")"); + } + + auto union_base = *reinterpret_cast(item.ptr); + + auto count = union_count < tag_vector->size() ? union_count : tag_vector->size(); + for (size_t i = 0; i < count; i++, union_base = PTR_ADD(union_base, union_type->byte_size())) + { + auto item_field = &union_type->getFields()[tag_vector->at(i) ? 1 : 0]; + ToCheck tagged_union_item(item, stl_sprintf("[%zu].%s", i, item_field->name), union_base, item_field->type); + queue_field(std::move(tagged_union_item), item_field); + } +} + void Checker::check_dispatch(ToCheck & item) { if (reinterpret_cast(item.ptr) == UNINIT_PTR) @@ -1267,7 +1301,7 @@ void Checker::check_bitarray(const ToCheck & item) // TODO: check DFHack::BitArray? } -void Checker::check_bitvector(const ToCheck & item) +bool Checker::check_bitvector(const ToCheck & item) { struct biterator_data { @@ -1285,10 +1319,11 @@ void Checker::check_bitvector(const ToCheck & item) if (item.identity->byte_size() != sizeof(bvector_data)) { UNEXPECTED; - return; + return false; } // TODO: check vector? + return true; } void Checker::check_struct(const ToCheck & item) From 61aeaaf55ef1517b1af84e008d08755605583b63 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Fri, 6 Mar 2020 16:50:09 -0600 Subject: [PATCH 05/32] fix typo in check-structures-sanity --- plugins/devel/check-structures-sanity.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/plugins/devel/check-structures-sanity.cpp b/plugins/devel/check-structures-sanity.cpp index 3a454a0e6..b4ed7f6c7 100644 --- a/plugins/devel/check-structures-sanity.cpp +++ b/plugins/devel/check-structures-sanity.cpp @@ -692,14 +692,17 @@ void Checker::queue_union(const ToCheck & item, const ToCheck & tag_item) void Checker::queue_union_vector(const ToCheck & item, const ToCheck & tag_item) { - if (tag_item.identity->getFullName(nullptr) == "vector") + auto union_container_type = static_cast(item.identity); + auto tag_container_type = static_cast(tag_item.identity); + + if (tag_container_type->getFullName(nullptr) == "vector") { queue_union_bitvector(item, tag_item); return; } - auto union_type = static_cast(static_cast(item.identity)->getItemType()); - auto tag_type = static_cast(static_cast(tag_item.identity)->getItemType()); + auto union_type = static_cast(union_container_type->getItemType()); + auto tag_type = static_cast(tag_container_type->getItemType()); auto union_count = check_vector_size(item, union_type->byte_size()); auto tag_count = check_vector_size(tag_item, tag_type->byte_size()); @@ -734,7 +737,7 @@ void Checker::queue_union_bitvector(const ToCheck & item, const ToCheck & tag_it if (union_count != tag_vector->size()) { - FAIL("tagged union vector size (" << union_count << ") does not match tag vector (" << join_strings("", tag_item.path) << ") size (" << tag_count << ")"); + FAIL("tagged union vector size (" << union_count << ") does not match tag vector (" << join_strings("", tag_item.path) << ") size (" << tag_vector->size() << ")"); } auto union_base = *reinterpret_cast(item.ptr); From def86b8058c1547dab089639996bc804d9f22336 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Fri, 6 Mar 2020 17:05:36 -0600 Subject: [PATCH 06/32] check-structures-sanity: ignore DfLinkedList element sizes; these can be part of a larger structure --- plugins/devel/check-structures-sanity.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/devel/check-structures-sanity.cpp b/plugins/devel/check-structures-sanity.cpp index b4ed7f6c7..8a5ae8b2e 100644 --- a/plugins/devel/check-structures-sanity.cpp +++ b/plugins/devel/check-structures-sanity.cpp @@ -1347,6 +1347,10 @@ void Checker::check_struct(const ToCheck & item) FAIL("allocated structure size (" << allocated_size << ") does not match expected size (" << expected_size << ")"); } } + else if (item.path.at(item.path.size() - 1) == "prev" || item.path.at(item.path.size() - 1) == "next") + { + // ignore unknown DfLinkedList sizes for now + } else { FAIL("unknown allocation size; possibly bad"); From 9b724666f78353189b7da7aa1f67f7bd2c37bf46 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Fri, 6 Mar 2020 17:53:05 -0600 Subject: [PATCH 07/32] check-structures-sanity: check linked lists in a more intelligent way --- plugins/devel/check-structures-sanity.cpp | 106 +++++++++++++++++++++- 1 file changed, 101 insertions(+), 5 deletions(-) diff --git a/plugins/devel/check-structures-sanity.cpp b/plugins/devel/check-structures-sanity.cpp index 8a5ae8b2e..ece998fc5 100644 --- a/plugins/devel/check-structures-sanity.cpp +++ b/plugins/devel/check-structures-sanity.cpp @@ -151,6 +151,7 @@ private: void queue_union(const ToCheck &, const ToCheck &); void queue_union_vector(const ToCheck &, const ToCheck &); void queue_union_bitvector(const ToCheck &, const ToCheck &); + void queue_df_linked_list(const ToCheck &); void check_dispatch(ToCheck &); void check_global(const ToCheck &); void check_primitive(const ToCheck &); @@ -551,7 +552,10 @@ void Checker::queue_field(ToCheck && item, const struct_field_info *field) queue.push_back(std::move(item)); break; case struct_field_info::CONTAINER: - queue.push_back(std::move(item)); + if (field->type && field->type->type() == IDTYPE_STRUCT) + queue_df_linked_list(item); + else + queue.push_back(std::move(item)); break; case struct_field_info::STL_VECTOR_PTR: item.temp_identity = std::unique_ptr(new df::stl_ptr_vector_identity(field->type, field->eid)); @@ -751,6 +755,96 @@ void Checker::queue_union_bitvector(const ToCheck & item, const ToCheck & tag_it } } +void Checker::queue_df_linked_list(const ToCheck & item) +{ + if (item.identity->type() != IDTYPE_STRUCT) + { + UNEXPECTED; + return; + } + + const struct_field_info *prev_field = nullptr, *next_field = nullptr, *item_field = nullptr; + auto fields = static_cast(item.identity)->getFields(); + for (auto field = fields; field->mode != struct_field_info::END; field++) + { +#define WANT_FIELD(fieldname, sametype) \ + if (!strcmp(#fieldname, field->name)) \ + { \ + if (field->mode != struct_field_info::POINTER) \ + { \ + FAIL("internal error: expected field " #fieldname " to be a pointer"); \ + UNEXPECTED; \ + return; \ + } \ + if (sametype && field->type != item.identity) \ + { \ + FAIL("internal error: expected field " #fieldname " to have a matching type"); \ + UNEXPECTED; \ + return; \ + } \ + if (fieldname ## _field) \ + { \ + FAIL("internal error: duplicate field " #fieldname); \ + UNEXPECTED; \ + return; \ + } \ + fieldname ## _field = field; \ + continue; \ + } + + WANT_FIELD(prev, true); + WANT_FIELD(next, true); + WANT_FIELD(item, false); + + FAIL("internal error: unexpected extra field " << field->name); + UNEXPECTED; + return; + } + + if (!prev_field || !next_field || !item_field) + { + FAIL("internal error: missing field(s) in DfLinkedList"); + UNEXPECTED; + return; + } + + // static verification of the type succeeded; continue to the actual list. + + int index = -1; + void *prev_ptr = nullptr; + void *cur_ptr = item.ptr; + + while (cur_ptr) + { + auto cur_prev_ptr = *reinterpret_cast(PTR_ADD(cur_ptr, prev_field->offset)); + if (prev_ptr != cur_prev_ptr) + { + FAIL("linked list element " << index << " previous element pointer " << stl_sprintf("%p", cur_prev_ptr) << " does not match actual previous element " << stl_sprintf("%p", prev_ptr)); + return; + } + + auto item_ptr_ptr = PTR_ADD(cur_ptr, item_field->offset); + std::unique_ptr item_ptr_identity(new df::pointer_identity(item_field->type)); + ToCheck item_item(item, stl_sprintf("[%d].item", index), item_ptr_ptr, item_ptr_identity.get()); + item_item.temp_identity = std::move(item_ptr_identity); + queue.push_back(std::move(item_item)); + + auto next_ptr = *reinterpret_cast(PTR_ADD(cur_ptr, next_field->offset)); + ToCheck next_item(item, stl_sprintf("[%d].next", index), next_ptr, item.identity); + if (check_access(next_item, next_ptr, item.identity)) + { + prev_ptr = cur_ptr; + cur_ptr = next_ptr; + } + else + { + cur_ptr = nullptr; + } + + index++; + } +} + void Checker::check_dispatch(ToCheck & item) { if (reinterpret_cast(item.ptr) == UNINIT_PTR) @@ -1347,10 +1441,6 @@ void Checker::check_struct(const ToCheck & item) FAIL("allocated structure size (" << allocated_size << ") does not match expected size (" << expected_size << ")"); } } - else if (item.path.at(item.path.size() - 1) == "prev" || item.path.at(item.path.size() - 1) == "next") - { - // ignore unknown DfLinkedList sizes for now - } else { FAIL("unknown allocation size; possibly bad"); @@ -1373,6 +1463,12 @@ void Checker::check_struct(const ToCheck & item) continue; } + if (!strcmp(field->name, "link") && field->mode == struct_field_info::POINTER && field->type->getFullName() == item.identity->getFullName() + "_list_link") + { + // skip linked list pointers + continue; + } + ToCheck child(item, std::string(".") + field->name, PTR_ADD(item.ptr, field->offset), field->type); queue_field(std::move(child), field); From 07aceb1078bafaabca9d1cd55b3a0da30b486f7a Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Fri, 6 Mar 2020 19:28:14 -0600 Subject: [PATCH 08/32] correctly handle arrays of linked lists --- plugins/devel/check-structures-sanity.cpp | 108 ++++++++++------------ 1 file changed, 51 insertions(+), 57 deletions(-) diff --git a/plugins/devel/check-structures-sanity.cpp b/plugins/devel/check-structures-sanity.cpp index ece998fc5..a57a63ab5 100644 --- a/plugins/devel/check-structures-sanity.cpp +++ b/plugins/devel/check-structures-sanity.cpp @@ -73,6 +73,36 @@ static void build_size_table() } } +static bool is_df_linked_list(type_identity *type) +{ + if (type->type() != IDTYPE_STRUCT) + return false; + + auto struct_type = static_cast(type); + auto fields = struct_type->getFields(); + + if (fields[0].mode != struct_field_info::POINTER) + return false; + if (strcmp(fields[0].name, "item")) + return false; + + if (fields[1].mode != struct_field_info::POINTER) + return false; + if (fields[1].type != type) + return false; + if (strcmp(fields[1].name, "prev")) + return false; + + if (fields[2].mode != struct_field_info::POINTER) + return false; + if (fields[2].type != type) + return false; + if (strcmp(fields[2].name, "next")) + return false; + + return fields[3].mode == struct_field_info::END; +} + static const char *const *get_enum_item_key(enum_identity *identity, int64_t value) { size_t index; @@ -572,6 +602,7 @@ void Checker::queue_field(ToCheck && item, const struct_field_info *field) void Checker::queue_static_array(const ToCheck & array, void *base, type_identity *type, size_t count, bool pointer, enum_identity *ienum) { size_t size = pointer ? sizeof(void *) : type->byte_size(); + bool is_linked_list = type && is_df_linked_list(type); for (size_t i = 0; i < count; i++, base = PTR_ADD(base, size)) { @@ -600,6 +631,11 @@ void Checker::queue_static_array(const ToCheck & array, void *base, type_identit item.temp_identity = std::unique_ptr(new pointer_identity(type)); item.identity = item.temp_identity.get(); } + else if (is_linked_list) + { + queue_df_linked_list(item); + continue; + } queue.push_back(std::move(item)); } } @@ -757,84 +793,42 @@ void Checker::queue_union_bitvector(const ToCheck & item, const ToCheck & tag_it void Checker::queue_df_linked_list(const ToCheck & item) { - if (item.identity->type() != IDTYPE_STRUCT) + if (!is_df_linked_list(item.identity)) { UNEXPECTED; return; } - const struct_field_info *prev_field = nullptr, *next_field = nullptr, *item_field = nullptr; - auto fields = static_cast(item.identity)->getFields(); - for (auto field = fields; field->mode != struct_field_info::END; field++) - { -#define WANT_FIELD(fieldname, sametype) \ - if (!strcmp(#fieldname, field->name)) \ - { \ - if (field->mode != struct_field_info::POINTER) \ - { \ - FAIL("internal error: expected field " #fieldname " to be a pointer"); \ - UNEXPECTED; \ - return; \ - } \ - if (sametype && field->type != item.identity) \ - { \ - FAIL("internal error: expected field " #fieldname " to have a matching type"); \ - UNEXPECTED; \ - return; \ - } \ - if (fieldname ## _field) \ - { \ - FAIL("internal error: duplicate field " #fieldname); \ - UNEXPECTED; \ - return; \ - } \ - fieldname ## _field = field; \ - continue; \ - } - - WANT_FIELD(prev, true); - WANT_FIELD(next, true); - WANT_FIELD(item, false); - - FAIL("internal error: unexpected extra field " << field->name); - UNEXPECTED; - return; - } - - if (!prev_field || !next_field || !item_field) - { - FAIL("internal error: missing field(s) in DfLinkedList"); - UNEXPECTED; - return; - } - - // static verification of the type succeeded; continue to the actual list. + auto item_type = static_cast(item.identity)->getFields()[2].type; int index = -1; - void *prev_ptr = nullptr; - void *cur_ptr = item.ptr; + struct df_linked_list_entry + { + df_linked_list_entry *prev; + df_linked_list_entry *next; + void *item; + } *prev_ptr = nullptr, *cur_ptr = reinterpret_cast(item.ptr); while (cur_ptr) { - auto cur_prev_ptr = *reinterpret_cast(PTR_ADD(cur_ptr, prev_field->offset)); - if (prev_ptr != cur_prev_ptr) + if (prev_ptr != cur_ptr->prev) { - FAIL("linked list element " << index << " previous element pointer " << stl_sprintf("%p", cur_prev_ptr) << " does not match actual previous element " << stl_sprintf("%p", prev_ptr)); + FAIL("linked list element " << index << " previous element pointer " << stl_sprintf("%p", cur_ptr->prev) << " does not match actual previous element " << stl_sprintf("%p", prev_ptr)); return; } - auto item_ptr_ptr = PTR_ADD(cur_ptr, item_field->offset); - std::unique_ptr item_ptr_identity(new df::pointer_identity(item_field->type)); + auto item_ptr_ptr = reinterpret_cast(&cur_ptr->item); + std::unique_ptr item_ptr_identity(new df::pointer_identity(item_type)); ToCheck item_item(item, stl_sprintf("[%d].item", index), item_ptr_ptr, item_ptr_identity.get()); item_item.temp_identity = std::move(item_ptr_identity); queue.push_back(std::move(item_item)); - auto next_ptr = *reinterpret_cast(PTR_ADD(cur_ptr, next_field->offset)); + auto next_ptr = reinterpret_cast(cur_ptr->next); ToCheck next_item(item, stl_sprintf("[%d].next", index), next_ptr, item.identity); if (check_access(next_item, next_ptr, item.identity)) { prev_ptr = cur_ptr; - cur_ptr = next_ptr; + cur_ptr = cur_ptr->next; } else { @@ -1463,7 +1457,7 @@ void Checker::check_struct(const ToCheck & item) continue; } - if (!strcmp(field->name, "link") && field->mode == struct_field_info::POINTER && field->type->getFullName() == item.identity->getFullName() + "_list_link") + if (field->mode == struct_field_info::POINTER && is_df_linked_list(field->type)) { // skip linked list pointers continue; From 64650374325e62064e97375167c8349ff88705ae Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Fri, 6 Mar 2020 20:04:17 -0600 Subject: [PATCH 09/32] fix field order --- plugins/devel/check-structures-sanity.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/devel/check-structures-sanity.cpp b/plugins/devel/check-structures-sanity.cpp index a57a63ab5..a0d9f3cce 100644 --- a/plugins/devel/check-structures-sanity.cpp +++ b/plugins/devel/check-structures-sanity.cpp @@ -804,9 +804,9 @@ void Checker::queue_df_linked_list(const ToCheck & item) int index = -1; struct df_linked_list_entry { + void *item; df_linked_list_entry *prev; df_linked_list_entry *next; - void *item; } *prev_ptr = nullptr, *cur_ptr = reinterpret_cast(item.ptr); while (cur_ptr) From f8f7b52180440a0c41a76c36d50be2a349b6f6e0 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Fri, 6 Mar 2020 20:22:18 -0600 Subject: [PATCH 10/32] fix is_df_linked_list not checking if the type is null --- plugins/devel/check-structures-sanity.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/devel/check-structures-sanity.cpp b/plugins/devel/check-structures-sanity.cpp index a0d9f3cce..0f4a29fa9 100644 --- a/plugins/devel/check-structures-sanity.cpp +++ b/plugins/devel/check-structures-sanity.cpp @@ -75,7 +75,7 @@ static void build_size_table() static bool is_df_linked_list(type_identity *type) { - if (type->type() != IDTYPE_STRUCT) + if (!type || type->type() != IDTYPE_STRUCT) return false; auto struct_type = static_cast(type); From 5f83681fbc49eec0a1f49afedd61e16f2bc0fb84 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Sat, 7 Mar 2020 12:08:03 -0600 Subject: [PATCH 11/32] update scripts and structures --- library/xml | 2 +- scripts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/library/xml b/library/xml index 23e66dc0d..3655adcae 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit 23e66dc0d44287a63fd918bbb81681510850c37d +Subproject commit 3655adcae556851c550bd3efe005f15e6a8dce02 diff --git a/scripts b/scripts index f19655a4f..8b701749c 160000 --- a/scripts +++ b/scripts @@ -1 +1 @@ -Subproject commit f19655a4fc7cb89391cb484d04ba1dff81a11d6d +Subproject commit 8b701749c3ee551a3af860e9173edc26131263f1 From c4e9c8d29c5d56c87c18cc2b069f65ee0c6feaea Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Sat, 7 Mar 2020 15:15:02 -0600 Subject: [PATCH 12/32] update structures --- library/xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/xml b/library/xml index 3655adcae..8038cc692 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit 3655adcae556851c550bd3efe005f15e6a8dce02 +Subproject commit 8038cc692370471aa1151f81cdc621ac310f30e2 From ffb3c29cfc7b932779412ffcd7d82bc41ec40809 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Sun, 8 Mar 2020 00:12:48 -0600 Subject: [PATCH 13/32] simplify Graphic module. update structures. it appears that all this added complexity including an extra pointer dereference was to avoid including the vector header. --- library/include/modules/Graphic.h | 6 ++---- library/modules/Graphic.cpp | 34 ++++++++----------------------- library/xml | 2 +- 3 files changed, 11 insertions(+), 31 deletions(-) diff --git a/library/include/modules/Graphic.h b/library/include/modules/Graphic.h index 8ee9c33b0..9103ce2be 100644 --- a/library/include/modules/Graphic.h +++ b/library/include/modules/Graphic.h @@ -33,6 +33,7 @@ distribution. #include #include "Export.h" #include "Module.h" +#include namespace DFHack { @@ -71,8 +72,6 @@ namespace DFHack class DFHACK_EXPORT Graphic : public Module { public: - Graphic(); - ~Graphic(); bool Finish() { return true; @@ -82,8 +81,7 @@ namespace DFHack DFTileSurface* Call(int x, int y); private: - struct Private; - Private *d; + std::vector funcs; }; } diff --git a/library/modules/Graphic.cpp b/library/modules/Graphic.cpp index 1d84926e3..a12ddb37c 100644 --- a/library/modules/Graphic.cpp +++ b/library/modules/Graphic.cpp @@ -47,40 +47,22 @@ std::unique_ptr DFHack::createGraphic() return dts::make_unique(); } -struct Graphic::Private -{ - bool Started; - vector funcs; -}; - -Graphic::Graphic() -{ - d = new Private; - - d->Started = true; -} - -Graphic::~Graphic() -{ - delete d; -} - bool Graphic::Register(DFTileSurface* (*func)(int,int)) { - d->funcs.push_back(func); + funcs.push_back(func); return true; } bool Graphic::Unregister(DFTileSurface* (*func)(int,int)) { - if ( d->funcs.empty() ) return false; + if ( funcs.empty() ) return false; - vector::iterator it = d->funcs.begin(); - while ( it != d->funcs.end() ) + vector::iterator it = funcs.begin(); + while ( it != funcs.end() ) { if ( *it == func ) { - d->funcs.erase(it); + funcs.erase(it); return true; } it++; @@ -92,12 +74,12 @@ bool Graphic::Unregister(DFTileSurface* (*func)(int,int)) // This will return first DFTileSurface it can get (or NULL if theres none) DFTileSurface* Graphic::Call(int x, int y) { - if ( d->funcs.empty() ) return NULL; + if ( funcs.empty() ) return NULL; DFTileSurface* temp = NULL; - vector::iterator it = d->funcs.begin(); - while ( it != d->funcs.end() ) + vector::iterator it = funcs.begin(); + while ( it != funcs.end() ) { temp = (*it)(x,y); if ( temp != NULL ) diff --git a/library/xml b/library/xml index 8038cc692..b3016446e 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit 8038cc692370471aa1151f81cdc621ac310f30e2 +Subproject commit b3016446e60f69d43a9f41e7b2ba4331df14b5e6 From a2e34a3b7103f7645751911ca37230e7b1bdbb90 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Sun, 8 Mar 2020 00:21:18 -0600 Subject: [PATCH 14/32] fix check-structures-sanity picking the wrong type for linked list items --- plugins/devel/check-structures-sanity.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/devel/check-structures-sanity.cpp b/plugins/devel/check-structures-sanity.cpp index 0f4a29fa9..d7de56c88 100644 --- a/plugins/devel/check-structures-sanity.cpp +++ b/plugins/devel/check-structures-sanity.cpp @@ -799,7 +799,7 @@ void Checker::queue_df_linked_list(const ToCheck & item) return; } - auto item_type = static_cast(item.identity)->getFields()[2].type; + auto item_type = static_cast(item.identity)->getFields()[0].type; int index = -1; struct df_linked_list_entry From a7d263fa6780837a65f395fd21c2b91a0a99cb8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre-David=20B=C3=A9langer?= Date: Mon, 9 Mar 2020 14:12:04 -0400 Subject: [PATCH 15/32] getFullName of the container AND the item, also exclude container of char (#1515) * getFullName of the container AND the item, also exclude container of char * bit vector condition was backwards --- library/DataDefs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/DataDefs.cpp b/library/DataDefs.cpp index c071ad7d3..00c44d975 100644 --- a/library/DataDefs.cpp +++ b/library/DataDefs.cpp @@ -540,7 +540,7 @@ const struct_field_info *DFHack::find_union_tag(const struct_field_info *fields, } auto union_fields = ((struct_identity*)union_field->type)->getFields(); - if (tag_container_type->getFullName(nullptr) != "vector" && + if (tag_container_type->getFullName() == "vector" && union_fields[0].mode != struct_field_info::END && union_fields[1].mode != struct_field_info::END && union_fields[2].mode == struct_field_info::END) From 923581b14446fdab433f1f6f08951ee9f5f3720b Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Sun, 8 Mar 2020 13:54:58 -0500 Subject: [PATCH 16/32] fix ghidra script failing if strings overlapped (for example, load_min_version and version could point to the same region of memory) --- plugins/devel/check-structures-sanity.cpp | 53 ++++++++++++++++------- reversing/find_df_globals.java | 2 +- 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/plugins/devel/check-structures-sanity.cpp b/plugins/devel/check-structures-sanity.cpp index d7de56c88..7f648ad4a 100644 --- a/plugins/devel/check-structures-sanity.cpp +++ b/plugins/devel/check-structures-sanity.cpp @@ -173,8 +173,8 @@ private: const std::string *check_possible_stl_string_pointer(const void *const*); #endif bool check_access(const ToCheck &, void *, type_identity *); - bool check_access(const ToCheck &, void *, type_identity *, size_t); - const char *check_vtable(const ToCheck &, void *, type_identity *); + bool check_access(const ToCheck &, void *, type_identity *, size_t, bool = false); + const char *check_vtable(const ToCheck &, void *, type_identity *, bool = false); void queue_field(ToCheck &&, const struct_field_info *); void queue_static_array(const ToCheck &, void *, type_identity *, size_t, bool = false, enum_identity * = nullptr); bool maybe_queue_union(const ToCheck &, const struct_field_info *, const struct_field_info *); @@ -427,7 +427,7 @@ bool Checker::check_access(const ToCheck & item, void *base, type_identity *iden return check_access(item, base, identity, identity ? identity->byte_size() : 0); } -bool Checker::check_access(const ToCheck & item, void *base, type_identity *identity, size_t size) +bool Checker::check_access(const ToCheck & item, void *base, type_identity *identity, size_t size, bool quiet) { if (!base) { @@ -445,7 +445,10 @@ bool Checker::check_access(const ToCheck & item, void *base, type_identity *iden #endif if (reinterpret_cast(base) == UNINIT_PTR) { - FAIL_PTR("uninitialized pointer"); + if (!quiet) + { + FAIL_PTR("uninitialized pointer"); + } return false; } @@ -467,7 +470,10 @@ bool Checker::check_access(const ToCheck & item, void *base, type_identity *iden if (!range.valid || !range.read) { - FAIL_PTR("pointer to invalid memory range"); + if (!quiet) + { + FAIL_PTR("pointer to invalid memory range"); + } return false; } @@ -483,6 +489,11 @@ bool Checker::check_access(const ToCheck & item, void *base, type_identity *iden } } + if (quiet) + { + return false; + } + if (expected_start == base) { FAIL_PTR("pointer not in any mapped range"); @@ -495,14 +506,14 @@ bool Checker::check_access(const ToCheck & item, void *base, type_identity *iden #undef FAIL_PTR } -const char *Checker::check_vtable(const ToCheck & item, void *vtable, type_identity *identity) +const char *Checker::check_vtable(const ToCheck & item, void *vtable, type_identity *identity, bool quiet) { - if (!check_access(item, PTR_ADD(vtable, -ptrdiff_t(sizeof(void *))), identity, sizeof(void *))) + if (!check_access(item, PTR_ADD(vtable, -ptrdiff_t(sizeof(void *))), identity, sizeof(void *), quiet)) return nullptr; char **info = *(reinterpret_cast(vtable) - 1); #ifdef WIN32 - if (!check_access(item, PTR_ADD(info, 12), identity, 4)) + if (!check_access(item, PTR_ADD(info, 12), identity, 4, quiet)) return nullptr; #ifdef DFHACK64 @@ -516,7 +527,7 @@ const char *Checker::check_vtable(const ToCheck & item, void *vtable, type_ident char *name = reinterpret_cast(info) + 8; #endif #else - if (!check_access(item, info + 1, identity, sizeof(void *))) + if (!check_access(item, info + 1, identity, sizeof(void *), quiet)) return nullptr; char *name = *(info + 1); #endif @@ -530,7 +541,10 @@ const char *Checker::check_vtable(const ToCheck & item, void *vtable, type_ident if (!range.valid || !range.read) { - FAIL("pointer to invalid memory range"); + if (!quiet) + { + FAIL("pointer to invalid memory range"); + } return nullptr; } @@ -863,18 +877,18 @@ void Checker::check_dispatch(ToCheck & item) size_t allocated_size = *reinterpret_cast(PTR_ADD(item.ptr, -16)); FAIL("pointer to a block of " << allocated_size << " bytes of allocated memory"); + if (allocated_size >= MIN_SIZE_FOR_SUGGEST && known_types_by_size.count(allocated_size)) + { + FAIL("known types of this size: " << join_strings(", ", known_types_by_size.at(allocated_size))); + } - // check recursively if it might be a valid pointer - if (allocated_size == sizeof(void *)) + // check recursively if it's the right size for a pointer or if it starts with what might be a valid pointer + if (allocated_size == sizeof(void *) || (allocated_size > sizeof(void *) && check_access(item, item.ptr, df::identity_traits::get(), sizeof(void *), true))) { item.path.push_back(".?ptr?"); item.path.push_back(""); item.identity = df::identity_traits::get(); } - else if (allocated_size >= MIN_SIZE_FOR_SUGGEST && known_types_by_size.count(allocated_size)) - { - FAIL("known types of this size: " << join_strings(", ", known_types_by_size.at(allocated_size))); - } } #ifndef WIN32 else if (auto str = check_possible_stl_string_pointer(&item.ptr)) @@ -907,6 +921,8 @@ void Checker::check_dispatch(ToCheck & item) // special case for large_integer weirdness if (item.identity == df::identity_traits::get()) { + // it's 16 bytes on 64-bit linux due to a messy header in libgraphics + // but only the first 8 bytes are ever used item.identity = df::identity_traits::get(); } @@ -1433,6 +1449,11 @@ void Checker::check_struct(const ToCheck & item) if (allocated_size != expected_size) { FAIL("allocated structure size (" << allocated_size << ") does not match expected size (" << expected_size << ")"); + + if (allocated_size >= MIN_SIZE_FOR_SUGGEST && known_types_by_size.count(allocated_size)) + { + FAIL("known types of this size: " << join_strings(", ", known_types_by_size.at(allocated_size))); + } } } else diff --git a/reversing/find_df_globals.java b/reversing/find_df_globals.java index e526fa671..95c8e9159 100644 --- a/reversing/find_df_globals.java +++ b/reversing/find_df_globals.java @@ -61,7 +61,7 @@ public class find_df_globals extends GhidraScript { dataAddr = globalAddr.getNewAddress(mem.getInt(globalAddr.add(globalCount * ptrSize * 2 + ptrSize))); } - String name = StringDataInstance.getStringDataInstance(currentProgram.getListing().createData(nameAddr, TerminatedStringDataType.dataType)).getStringValue(); + String name = StringDataInstance.getStringDataInstance(DataUtilities.createData(currentProgram, nameAddr, TerminatedStringDataType.dataType, 0, false, DataUtilities.ClearDataMode.CLEAR_ALL_CONFLICT_DATA)).getStringValue(); createLabel(dataAddr, name, true); } From e296525983644fb38e7cff22ef64ede094f1595e Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Sun, 8 Mar 2020 14:24:40 -0500 Subject: [PATCH 17/32] check-structures-sanity: don't error on unnamed enum values/bits by default --- plugins/devel/check-structures-sanity.cpp | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/plugins/devel/check-structures-sanity.cpp b/plugins/devel/check-structures-sanity.cpp index 7f648ad4a..990324292 100644 --- a/plugins/devel/check-structures-sanity.cpp +++ b/plugins/devel/check-structures-sanity.cpp @@ -48,6 +48,7 @@ DFhackCExport command_result plugin_init(color_ostream &, std::vector & pa } BOOL_PARAM(enums); BOOL_PARAM(sizes); + BOOL_PARAM(unnamed); BOOL_PARAM(lowmem); BOOL_PARAM(failfast); #undef BOOL_PARAM @@ -303,6 +306,7 @@ Checker::Checker(color_ostream & out) : Core::getInstance().p->getMemRanges(mapped); enums = false; sizes = false; + unnamed = false; lowmem = false; failfast = false; maxerrors = ~size_t(0); @@ -684,7 +688,7 @@ void Checker::queue_union(const ToCheck & item, const ToCheck & tag_item) { FAIL("tagged union tag (" << join_strings("", tag_item.path) << ") out of range (" << tag_value << ")"); } - else if (!tag_key) + else if (!tag_key && unnamed) { FAIL("tagged union tag (" << join_strings("", tag_item.path) << ") unnamed (" << tag_value << ")"); } @@ -1160,8 +1164,12 @@ void Checker::check_bitfield(const ToCheck & item) size_t num_bits = identity->getNumBits(); auto bits = identity->getBits(); + size_t next_bit = 0; for (size_t i = 0; i < num_bits; i++) { + if (bits[i].size) + next_bit = i + 1; + if (bits[i].size < 0) continue; if (bits[i].name) @@ -1170,13 +1178,13 @@ void Checker::check_bitfield(const ToCheck & item) if (!(val & (1ULL << i))) continue; - if (bits[i].size) + if (!bits[i].size) { - FAIL("bitfield bit " << i << " is unnamed"); + FAIL("bitfield bit " << i << " past the defined end of the bitfield (" << next_bit << " would be next)"); } - else + else if (unnamed) { - FAIL("bitfield bit " << i << " past the defined end of the bitfield"); + FAIL("bitfield bit " << i << " is unnamed"); } } } @@ -1231,7 +1239,7 @@ int64_t Checker::check_enum(const ToCheck & item) FAIL("enum value (" << value << ") outside of defined range (" << identity->getFirstItem() << " to " << identity->getLastItem() << ")"); } } - else if (!*key) + else if (!*key && unnamed) { FAIL("enum value (" << value << ") is unnamed"); } From b9841110c3363765485e696dd9395efb981a2971 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Mon, 9 Mar 2020 17:53:34 -0500 Subject: [PATCH 18/32] update structures and scripts --- library/xml | 2 +- scripts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/library/xml b/library/xml index b3016446e..e2fafcd39 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit b3016446e60f69d43a9f41e7b2ba4331df14b5e6 +Subproject commit e2fafcd396f71a1b67f74e8f2e9b7c51d2fbadf3 diff --git a/scripts b/scripts index 8b701749c..a1ae6a095 160000 --- a/scripts +++ b/scripts @@ -1 +1 @@ -Subproject commit 8b701749c3ee551a3af860e9173edc26131263f1 +Subproject commit a1ae6a095b626c2667519176a28a40d5214e6fb9 From e5de783c58dec046740b4f93a42e91288d502a92 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Tue, 10 Mar 2020 18:53:56 -0500 Subject: [PATCH 19/32] rewriting check-structures-sanity to improve performance and remove the need for the lowmem option --- plugins/devel/CMakeLists.txt | 3 +- plugins/devel/check-structures-sanity.cpp | 1532 ----------------- .../check-structures-sanity/CMakeLists.txt | 8 + .../check-structures-sanity.h | 117 ++ .../check-structures-sanity/dispatch.cpp | 389 +++++ .../devel/check-structures-sanity/main.cpp | 126 ++ .../devel/check-structures-sanity/types.cpp | 131 ++ .../check-structures-sanity/validate.cpp | 237 +++ 8 files changed, 1010 insertions(+), 1533 deletions(-) delete mode 100644 plugins/devel/check-structures-sanity.cpp create mode 100644 plugins/devel/check-structures-sanity/CMakeLists.txt create mode 100644 plugins/devel/check-structures-sanity/check-structures-sanity.h create mode 100644 plugins/devel/check-structures-sanity/dispatch.cpp create mode 100644 plugins/devel/check-structures-sanity/main.cpp create mode 100644 plugins/devel/check-structures-sanity/types.cpp create mode 100644 plugins/devel/check-structures-sanity/validate.cpp diff --git a/plugins/devel/CMakeLists.txt b/plugins/devel/CMakeLists.txt index 7a0e855c7..6d15f09dd 100644 --- a/plugins/devel/CMakeLists.txt +++ b/plugins/devel/CMakeLists.txt @@ -6,7 +6,6 @@ include(FindThreads) add_definitions(-DDEV_PLUGIN) dfhack_plugin(buildprobe buildprobe.cpp) -dfhack_plugin(check-structures-sanity check-structures-sanity.cpp LINK_LIBRARIES lua) dfhack_plugin(color-dfhack-text color-dfhack-text.cpp) dfhack_plugin(counters counters.cpp) dfhack_plugin(dumpmats dumpmats.cpp) @@ -27,3 +26,5 @@ dfhack_plugin(zoom zoom.cpp) if(UNIX) dfhack_plugin(ref-index ref-index.cpp) endif() + +add_subdirectory(check-structures-sanity) diff --git a/plugins/devel/check-structures-sanity.cpp b/plugins/devel/check-structures-sanity.cpp deleted file mode 100644 index 990324292..000000000 --- a/plugins/devel/check-structures-sanity.cpp +++ /dev/null @@ -1,1532 +0,0 @@ -#include "Console.h" -#include "PluginManager.h" -#include "MemAccess.h" -#include "DataDefs.h" -#include "DataIdentity.h" -#include "LuaTools.h" -#include "LuaWrapper.h" - -#include "df/large_integer.h" - -#if defined(WIN32) && defined(DFHACK64) -#define _WIN32_WINNT 0x0501 -#define WINVER 0x0501 - -#define WIN32_LEAN_AND_MEAN -#include -#endif - -#include -#include -#include -#include - -using namespace DFHack; - -DFHACK_PLUGIN("check-structures-sanity"); - -static command_result command(color_ostream &, std::vector &); - -#ifdef WIN32 -#define UNEXPECTED __debugbreak() -#else -#define UNEXPECTED __asm__ volatile ("int $0x03") -#endif - -#define MIN_SIZE_FOR_SUGGEST 64 -static std::map> known_types_by_size; -static void build_size_table(); - -DFhackCExport command_result plugin_init(color_ostream &, std::vector & commands) -{ - commands.push_back(PluginCommand( - "check-structures-sanity", - "performs a sanity check on df-structures", - command, - false, - "check-structures-sanity [-enums] [-sizes] [-lowmem] [-maxerrors n] [-failfast] [starting_point]\n" - "\n" - "-enums: report unexpected or unnamed enum or bitfield values.\n" - "-sizes: report struct and class sizes that don't match structures. (requires sizecheck)\n" - "-unnamed: report unnamed enum/bitfield values, not just undefined ones.\n" - "-lowmem: use depth-first search instead of breadth-first search. uses less memory but processes fields in a less intuitive order.\n" - "-maxerrors n: set the maximum number of errors before bailing out.\n" - "-failfast: crash if any error is encountered. useful only for debugging.\n" - "starting_point: a lua expression or a word like 'screen', 'item', or 'building'. (defaults to df.global)\n" - "\n" - "by default, check-structures-sanity reports invalid pointers, vectors, strings, and vtables." - )); - - known_types_by_size.clear(); - build_size_table(); - - return CR_OK; -} - -static void build_size_table() -{ - for (auto & ident : compound_identity::getTopScope()) - { - if (ident->byte_size() >= MIN_SIZE_FOR_SUGGEST) - { - known_types_by_size[ident->byte_size()].push_back(ident->getFullName()); - } - } -} - -static bool is_df_linked_list(type_identity *type) -{ - if (!type || type->type() != IDTYPE_STRUCT) - return false; - - auto struct_type = static_cast(type); - auto fields = struct_type->getFields(); - - if (fields[0].mode != struct_field_info::POINTER) - return false; - if (strcmp(fields[0].name, "item")) - return false; - - if (fields[1].mode != struct_field_info::POINTER) - return false; - if (fields[1].type != type) - return false; - if (strcmp(fields[1].name, "prev")) - return false; - - if (fields[2].mode != struct_field_info::POINTER) - return false; - if (fields[2].type != type) - return false; - if (strcmp(fields[2].name, "next")) - return false; - - return fields[3].mode == struct_field_info::END; -} - -static const char *const *get_enum_item_key(enum_identity *identity, int64_t value) -{ - size_t index; - if (auto cplx = identity->getComplex()) - { - auto it = cplx->value_index_map.find(value); - if (it == cplx->value_index_map.cend()) - { - return nullptr; - } - index = it->second; - } - else - { - if (value < identity->getFirstItem() || value > identity->getLastItem()) - { - return nullptr; - } - index = value - identity->getFirstItem(); - } - - return &identity->getKeys()[index]; -} - -struct ToCheck -{ - std::vector path; - void *ptr; - type_identity *identity; - std::unique_ptr temp_identity; - - ToCheck() - { - } - - ToCheck(const ToCheck & parent, size_t idx, void *ptr, type_identity *identity) : - ToCheck(parent, stl_sprintf("[%zu]", idx), ptr, identity) - { - } - - ToCheck(const ToCheck & parent, const std::string & name, void *ptr, type_identity *identity) : - path(parent.path.cbegin(), parent.path.cend()), - ptr(ptr), - identity(identity) - { - path.push_back(name); - } -}; - -class Checker -{ - color_ostream & out; - std::vector mapped; - std::set seen_addr; -public: - std::deque queue; - size_t num_checked; - bool enums; - bool sizes; - bool unnamed; - bool lowmem; - bool failfast; - size_t maxerrors; -private: - bool ok; - -#ifndef WIN32 - // this function doesn't make sense on windows, where std::string is not pointer-sized. - const std::string *check_possible_stl_string_pointer(const void *const*); -#endif - bool check_access(const ToCheck &, void *, type_identity *); - bool check_access(const ToCheck &, void *, type_identity *, size_t, bool = false); - const char *check_vtable(const ToCheck &, void *, type_identity *, bool = false); - void queue_field(ToCheck &&, const struct_field_info *); - void queue_static_array(const ToCheck &, void *, type_identity *, size_t, bool = false, enum_identity * = nullptr); - bool maybe_queue_union(const ToCheck &, const struct_field_info *, const struct_field_info *); - void queue_union(const ToCheck &, const ToCheck &); - void queue_union_vector(const ToCheck &, const ToCheck &); - void queue_union_bitvector(const ToCheck &, const ToCheck &); - void queue_df_linked_list(const ToCheck &); - void check_dispatch(ToCheck &); - void check_global(const ToCheck &); - void check_primitive(const ToCheck &); - void check_stl_string(const ToCheck &); - void check_pointer(const ToCheck &); - void check_bitfield(const ToCheck &); - int64_t check_enum(const ToCheck &); - void check_container(const ToCheck &); - size_t check_vector_size(const ToCheck &, size_t); - void check_vector(const ToCheck &, type_identity *, bool); - void check_deque(const ToCheck &, type_identity *); - void check_dfarray(const ToCheck &, type_identity *); - void check_bitarray(const ToCheck &); - bool check_bitvector(const ToCheck &); - void check_struct(const ToCheck &); - void check_virtual(const ToCheck &); -public: - Checker(color_ostream &); - bool check(); -}; - -static command_result command(color_ostream & out, std::vector & parameters) -{ - CoreSuspender suspend; - - Checker checker(out); - - // check parameters with values first -#define VAL_PARAM(name, expr_using_value) \ - auto name ## _idx = std::find(parameters.begin(), parameters.end(), "-" #name); \ - if (name ## _idx != parameters.end()) \ - { \ - if (name ## _idx + 1 == parameters.end()) \ - { \ - return CR_WRONG_USAGE; \ - } \ - try \ - { \ - auto value = std::move(*(name ## _idx + 1)); \ - parameters.erase((name ## _idx + 1)); \ - parameters.erase(name ## _idx); \ - checker.name = (expr_using_value); \ - } \ - catch (std::exception & ex) \ - { \ - out.printerr("check-structures-sanity: argument to -%s: %s\n", #name, ex.what()); \ - return CR_WRONG_USAGE; \ - } \ - } - VAL_PARAM(maxerrors, std::stoul(value)); -#undef VAL_PARAM - -#define BOOL_PARAM(name) \ - auto name ## _idx = std::find(parameters.begin(), parameters.end(), "-" #name); \ - if (name ## _idx != parameters.end()) \ - { \ - checker.name = true; \ - parameters.erase(name ## _idx); \ - } - BOOL_PARAM(enums); - BOOL_PARAM(sizes); - BOOL_PARAM(unnamed); - BOOL_PARAM(lowmem); - BOOL_PARAM(failfast); -#undef BOOL_PARAM - - if (parameters.size() > 1) - { - return CR_WRONG_USAGE; - } - - if (parameters.empty()) - { - ToCheck global; - global.path.push_back("df.global."); - global.ptr = nullptr; - global.identity = &df::global::_identity; - - checker.queue.push_back(std::move(global)); - } - else - { - using namespace DFHack::Lua; - using namespace DFHack::Lua::Core; - using namespace DFHack::LuaWrapper; - - StackUnwinder unwinder(State); - PushModulePublic(out, "utils", "df_expr_to_ref"); - Push(parameters.at(0)); - if (!SafeCall(out, 1, 1)) - { - return CR_FAILURE; - } - if (!lua_touserdata(State, -1)) - { - return CR_WRONG_USAGE; - } - - ToCheck ref; - ref.path.push_back(parameters.at(0)); - ref.ptr = get_object_ref(State, -1); - lua_getfield(State, -1, "_type"); - lua_getfield(State, -1, "_identity"); - ref.identity = reinterpret_cast(lua_touserdata(State, -1)); - if (!ref.identity) - { - out.printerr("could not determine type identity\n"); - return CR_FAILURE; - } - - checker.queue.push_back(std::move(ref)); - } - - return checker.check() ? CR_OK : CR_FAILURE; -} - -Checker::Checker(color_ostream & out) : - out(out) -{ - Core::getInstance().p->getMemRanges(mapped); - enums = false; - sizes = false; - unnamed = false; - lowmem = false; - failfast = false; - maxerrors = ~size_t(0); -} - -bool Checker::check() -{ - seen_addr.clear(); - num_checked = 0; - ok = true; - - while (!queue.empty()) - { - if (!maxerrors) - { - out << "hit max error count. bailing out with " << queue.size() << " fields in queue." << std::endl; - break; - } - - ToCheck current; - if (lowmem) - { - current = std::move(queue.back()); - queue.pop_back(); - } - else - { - current = std::move(queue.front()); - queue.pop_front(); - } - - check_dispatch(current); - - num_checked++; - if (out.is_console() && num_checked % 1000 == 0) - { - out << "checked " << num_checked << " fields\r" << std::flush; - } - } - - out << "checked " << num_checked << " fields" << std::endl; - - return ok; -} - -#define FAIL(message) \ - do \ - { \ - ok = false; \ - out << COLOR_LIGHTRED << "sanity check failed (line " << __LINE__ << "): "; \ - out << COLOR_RESET << (item.identity ? item.identity->getFullName() : "?") << " (accessed as "; \ - for (auto & p : item.path) { out << p; } \ - out << "): "; \ - out << COLOR_YELLOW << message; \ - out << COLOR_RESET << std::endl; \ - if (maxerrors && maxerrors != ~size_t(0)) \ - maxerrors--; \ - if (failfast) \ - UNEXPECTED; \ - } while (false) - -#define PTR_ADD(base, offset) (reinterpret_cast(reinterpret_cast((base)) + static_cast((offset)))) - -#ifndef WIN32 -const std::string *Checker::check_possible_stl_string_pointer(const void *const*base) -{ - std::string empty_string; - if (*base == *reinterpret_cast(&empty_string)) - { - return reinterpret_cast(base); - } - - const struct string_data_inner - { - size_t length; - size_t capacity; - int32_t refcount; - } *str_data = static_cast(*base) - 1; - - uint32_t tag = *reinterpret_cast(PTR_ADD(str_data, -8)); - if (tag == 0xdfdf4ac8) - { - size_t allocated_size = *reinterpret_cast(PTR_ADD(str_data, -16)); - size_t expected_size = sizeof(*str_data) + str_data->capacity + 1; - - if (allocated_size != expected_size) - { - return nullptr; - } - } - else - { - return nullptr; - } - - if (str_data->capacity < str_data->length) - { - return nullptr; - } - - const char *ptr = reinterpret_cast(*base); - for (size_t i = 0; i < str_data->length; i++) - { - if (!*ptr++) - { - return nullptr; - } - } - - if (*ptr++) - { - return nullptr; - } - - return reinterpret_cast(base); -} -#endif - - -bool Checker::check_access(const ToCheck & item, void *base, type_identity *identity) -{ - return check_access(item, base, identity, identity ? identity->byte_size() : 0); -} - -bool Checker::check_access(const ToCheck & item, void *base, type_identity *identity, size_t size, bool quiet) -{ - if (!base) - { - // null pointer: can't access, but not an error - return false; - } - - // assumes MALLOC_PERTURB_=45 -#ifdef DFHACK64 -#define UNINIT_PTR 0xd2d2d2d2d2d2d2d2 -#define FAIL_PTR(message) FAIL(stl_sprintf("0x%016zx: ", reinterpret_cast(base)) << message) -#else -#define UNINIT_PTR 0xd2d2d2d2 -#define FAIL_PTR(message) FAIL(stl_sprintf("0x%08zx: ", reinterpret_cast(base)) << message) -#endif - if (reinterpret_cast(base) == UNINIT_PTR) - { - if (!quiet) - { - FAIL_PTR("uninitialized pointer"); - } - return false; - } - - bool found = true; - void *expected_start = base; - size_t remaining_size = size; - while (found) - { - found = false; - - for (auto & range : mapped) - { - if (!range.isInRange(expected_start)) - { - continue; - } - - found = true; - - if (!range.valid || !range.read) - { - if (!quiet) - { - FAIL_PTR("pointer to invalid memory range"); - } - return false; - } - - if (size && !range.isInRange(PTR_ADD(expected_start, remaining_size - 1))) - { - void *next_start = PTR_ADD(range.end, 1); - remaining_size -= reinterpret_cast(next_start) - reinterpret_cast(expected_start); - expected_start = next_start; - break; - } - - return true; - } - } - - if (quiet) - { - return false; - } - - if (expected_start == base) - { - FAIL_PTR("pointer not in any mapped range"); - } - else - { - FAIL_PTR("pointer exceeds mapped memory bounds (size " << size << ")"); - } - return false; -#undef FAIL_PTR -} - -const char *Checker::check_vtable(const ToCheck & item, void *vtable, type_identity *identity, bool quiet) -{ - if (!check_access(item, PTR_ADD(vtable, -ptrdiff_t(sizeof(void *))), identity, sizeof(void *), quiet)) - return nullptr; - char **info = *(reinterpret_cast(vtable) - 1); - -#ifdef WIN32 - if (!check_access(item, PTR_ADD(info, 12), identity, 4, quiet)) - return nullptr; - -#ifdef DFHACK64 - void *base; - if (!RtlPcToFileHeader(info, &base)) - return nullptr; - - char *typeinfo = reinterpret_cast(base) + reinterpret_cast(info)[3]; - char *name = typeinfo + 16; -#else - char *name = reinterpret_cast(info) + 8; -#endif -#else - if (!check_access(item, info + 1, identity, sizeof(void *), quiet)) - return nullptr; - char *name = *(info + 1); -#endif - - for (auto & range : mapped) - { - if (!range.isInRange(name)) - { - continue; - } - - if (!range.valid || !range.read) - { - if (!quiet) - { - FAIL("pointer to invalid memory range"); - } - return nullptr; - } - - bool letter = false; - for (char *p = name; ; p++) - { - if (!range.isInRange(p)) - { - return nullptr; - } - - if (*p >= 'a' && *p <= 'z') - { - letter = true; - } - else if (!*p) - { - return letter ? name : nullptr; - } - } - } - - return nullptr; -} - -void Checker::queue_field(ToCheck && item, const struct_field_info *field) -{ - switch (field->mode) - { - case struct_field_info::END: - UNEXPECTED; - break; - case struct_field_info::PRIMITIVE: - queue.push_back(std::move(item)); - break; - case struct_field_info::STATIC_STRING: - // TODO: check static strings? - break; - case struct_field_info::POINTER: - // TODO: flags inside field->count - item.temp_identity = std::unique_ptr(new df::pointer_identity(field->type)); - item.identity = item.temp_identity.get(); - queue.push_back(std::move(item)); - break; - case struct_field_info::STATIC_ARRAY: - queue_static_array(item, item.ptr, field->type, field->count, false, field->eid); - break; - case struct_field_info::SUBSTRUCT: - queue.push_back(std::move(item)); - break; - case struct_field_info::CONTAINER: - if (field->type && field->type->type() == IDTYPE_STRUCT) - queue_df_linked_list(item); - else - queue.push_back(std::move(item)); - break; - case struct_field_info::STL_VECTOR_PTR: - item.temp_identity = std::unique_ptr(new df::stl_ptr_vector_identity(field->type, field->eid)); - item.identity = item.temp_identity.get(); - queue.push_back(std::move(item)); - break; - case struct_field_info::OBJ_METHOD: - case struct_field_info::CLASS_METHOD: - // ignore - break; - } -} - -void Checker::queue_static_array(const ToCheck & array, void *base, type_identity *type, size_t count, bool pointer, enum_identity *ienum) -{ - size_t size = pointer ? sizeof(void *) : type->byte_size(); - bool is_linked_list = type && is_df_linked_list(type); - - for (size_t i = 0; i < count; i++, base = PTR_ADD(base, size)) - { - ToCheck item(array, i, base, type); - if (ienum) - { - auto pname = get_enum_item_key(ienum, int64_t(i)); - auto name = pname ? *pname : nullptr; - - std::ostringstream str; - str << "[" << ienum->getFullName() << "::"; - if (name) - { - str << name; - } - else - { - str << "?" << i << "?"; - } - str << "]"; - - item.path.back() = str.str(); - } - if (pointer) - { - item.temp_identity = std::unique_ptr(new pointer_identity(type)); - item.identity = item.temp_identity.get(); - } - else if (is_linked_list) - { - queue_df_linked_list(item); - continue; - } - queue.push_back(std::move(item)); - } -} - -bool Checker::maybe_queue_union(const ToCheck & item, const struct_field_info *fields, const struct_field_info *union_field) -{ - auto tag_field = find_union_tag(fields, union_field); - if (!tag_field) - return false; - - ToCheck union_item(item, "." + std::string(union_field->name), PTR_ADD(item.ptr, union_field->offset), union_field->type); - ToCheck tag_item(item, "." + std::string(tag_field->name), PTR_ADD(item.ptr, tag_field->offset), tag_field->type); - - if (union_field->mode == struct_field_info::SUBSTRUCT) - queue_union(union_item, tag_item); - else - queue_union_vector(union_item, tag_item); - - return true; -} - -void Checker::queue_union(const ToCheck & item, const ToCheck & tag_item) -{ - auto union_type = static_cast(item.identity); - auto tag_type = static_cast(tag_item.identity); - - int64_t tag_value = check_enum(tag_item); - - auto ptag_key = get_enum_item_key(tag_type, tag_value); - auto tag_key = ptag_key ? *ptag_key : nullptr; - if (!ptag_key) - { - FAIL("tagged union tag (" << join_strings("", tag_item.path) << ") out of range (" << tag_value << ")"); - } - else if (!tag_key && unnamed) - { - FAIL("tagged union tag (" << join_strings("", tag_item.path) << ") unnamed (" << tag_value << ")"); - } - - const struct_field_info *item_field = nullptr; - if (tag_key) - { - for (auto field = union_type->getFields(); field->mode != struct_field_info::END; field++) - { - if (!strcmp(tag_key, field->name)) - { - item_field = field; - break; - } - } - } - - if (item_field) - { - // good to go - ToCheck tagged_union_item(item, "." + std::string(item_field->name), item.ptr, item_field->type); - queue_field(std::move(tagged_union_item), item_field); - return; - } - - // if it's all uninitialized, ignore it - uint8_t uninit_value = *reinterpret_cast(item.ptr); - bool all_uninitialized = uninit_value == 0x00 || uninit_value == 0xd2 || uninit_value == 0xff; - if (all_uninitialized) - { - for (size_t offset = 0; offset < union_type->byte_size(); offset++) - { - if (*reinterpret_cast(PTR_ADD(item.ptr, offset)) != uninit_value) - { - all_uninitialized = false; - break; - } - } - } - if (all_uninitialized) - { - return; - } - - // if we don't know the key, we already warned above - if (tag_key) - { - FAIL("tagged union (" << join_strings("", tag_item.path) << ") missing member for tag " << tag_key << " (" << tag_value << ")"); - } - - // if there's a pointer (we only check the first field for now) - // assume this could also be a pointer - if (union_type->getFields()->mode == struct_field_info::POINTER) - { - ToCheck untagged_union_item(item, tag_key ? "." + std::string(tag_key) : stl_sprintf(".?%" PRId64 "?", tag_value), item.ptr, df::identity_traits::get()); - queue.push_back(std::move(untagged_union_item)); - } -} - -void Checker::queue_union_vector(const ToCheck & item, const ToCheck & tag_item) -{ - auto union_container_type = static_cast(item.identity); - auto tag_container_type = static_cast(tag_item.identity); - - if (tag_container_type->getFullName(nullptr) == "vector") - { - queue_union_bitvector(item, tag_item); - return; - } - - auto union_type = static_cast(union_container_type->getItemType()); - auto tag_type = static_cast(tag_container_type->getItemType()); - - auto union_count = check_vector_size(item, union_type->byte_size()); - auto tag_count = check_vector_size(tag_item, tag_type->byte_size()); - - if (union_count != tag_count) - { - FAIL("tagged union vector size (" << union_count << ") does not match tag vector (" << join_strings("", tag_item.path) << ") size (" << tag_count << ")"); - } - - auto union_base = *reinterpret_cast(item.ptr); - auto tag_base = *reinterpret_cast(tag_item.ptr); - - auto count = union_count < tag_count ? union_count : tag_count; - for (size_t i = 0; i < count; i++, union_base = PTR_ADD(union_base, union_type->byte_size()), tag_base = PTR_ADD(tag_base, tag_type->byte_size())) - { - ToCheck union_item(item, i, union_base, union_type); - ToCheck tag(tag_item, i, tag_base, tag_type); - queue_union(union_item, tag); - } -} - -void Checker::queue_union_bitvector(const ToCheck & item, const ToCheck & tag_item) -{ - auto union_type = static_cast(static_cast(item.identity)->getItemType()); - auto union_count = check_vector_size(item, union_type->byte_size()); - - if (!check_bitvector(tag_item)) - { - return; - } - auto tag_vector = reinterpret_cast *>(tag_item.ptr); - - if (union_count != tag_vector->size()) - { - FAIL("tagged union vector size (" << union_count << ") does not match tag vector (" << join_strings("", tag_item.path) << ") size (" << tag_vector->size() << ")"); - } - - auto union_base = *reinterpret_cast(item.ptr); - - auto count = union_count < tag_vector->size() ? union_count : tag_vector->size(); - for (size_t i = 0; i < count; i++, union_base = PTR_ADD(union_base, union_type->byte_size())) - { - auto item_field = &union_type->getFields()[tag_vector->at(i) ? 1 : 0]; - ToCheck tagged_union_item(item, stl_sprintf("[%zu].%s", i, item_field->name), union_base, item_field->type); - queue_field(std::move(tagged_union_item), item_field); - } -} - -void Checker::queue_df_linked_list(const ToCheck & item) -{ - if (!is_df_linked_list(item.identity)) - { - UNEXPECTED; - return; - } - - auto item_type = static_cast(item.identity)->getFields()[0].type; - - int index = -1; - struct df_linked_list_entry - { - void *item; - df_linked_list_entry *prev; - df_linked_list_entry *next; - } *prev_ptr = nullptr, *cur_ptr = reinterpret_cast(item.ptr); - - while (cur_ptr) - { - if (prev_ptr != cur_ptr->prev) - { - FAIL("linked list element " << index << " previous element pointer " << stl_sprintf("%p", cur_ptr->prev) << " does not match actual previous element " << stl_sprintf("%p", prev_ptr)); - return; - } - - auto item_ptr_ptr = reinterpret_cast(&cur_ptr->item); - std::unique_ptr item_ptr_identity(new df::pointer_identity(item_type)); - ToCheck item_item(item, stl_sprintf("[%d].item", index), item_ptr_ptr, item_ptr_identity.get()); - item_item.temp_identity = std::move(item_ptr_identity); - queue.push_back(std::move(item_item)); - - auto next_ptr = reinterpret_cast(cur_ptr->next); - ToCheck next_item(item, stl_sprintf("[%d].next", index), next_ptr, item.identity); - if (check_access(next_item, next_ptr, item.identity)) - { - prev_ptr = cur_ptr; - cur_ptr = cur_ptr->next; - } - else - { - cur_ptr = nullptr; - } - - index++; - } -} - -void Checker::check_dispatch(ToCheck & item) -{ - if (reinterpret_cast(item.ptr) == UNINIT_PTR) - { - // allow uninitialized raw pointers - return; - } - - if (!item.identity) - { - // warn about bad pointers - if (!check_access(item, item.ptr, df::identity_traits::get(), 1)) - { - return; - } - - if (sizes) - { - uint32_t tag = *reinterpret_cast(PTR_ADD(item.ptr, -8)); - if (tag == 0xdfdf4ac8) - { - size_t allocated_size = *reinterpret_cast(PTR_ADD(item.ptr, -16)); - - FAIL("pointer to a block of " << allocated_size << " bytes of allocated memory"); - if (allocated_size >= MIN_SIZE_FOR_SUGGEST && known_types_by_size.count(allocated_size)) - { - FAIL("known types of this size: " << join_strings(", ", known_types_by_size.at(allocated_size))); - } - - // check recursively if it's the right size for a pointer or if it starts with what might be a valid pointer - if (allocated_size == sizeof(void *) || (allocated_size > sizeof(void *) && check_access(item, item.ptr, df::identity_traits::get(), sizeof(void *), true))) - { - item.path.push_back(".?ptr?"); - item.path.push_back(""); - item.identity = df::identity_traits::get(); - } - } -#ifndef WIN32 - else if (auto str = check_possible_stl_string_pointer(&item.ptr)) - { - FAIL("untyped pointer is actually stl-string with value \"" << *str << "\" (length " << str->length() << ")"); - } -#endif - else if (auto vtable_name = check_vtable(item, item.ptr, df::identity_traits::get())) - { - FAIL("pointer to a vtable: " << vtable_name); - } - else - { - FAIL("pointer to memory with no size information"); - } - } - - // could have been set above - if (!item.identity) - { - return; - } - } - - if (!check_access(item, item.ptr, item.identity) && item.identity->type() != IDTYPE_GLOBAL) - { - return; - } - - // special case for large_integer weirdness - if (item.identity == df::identity_traits::get()) - { - // it's 16 bytes on 64-bit linux due to a messy header in libgraphics - // but only the first 8 bytes are ever used - item.identity = df::identity_traits::get(); - } - - - switch (item.identity->type()) - { - case IDTYPE_GLOBAL: - check_global(item); - break; - case IDTYPE_FUNCTION: - // don't check functions - break; - case IDTYPE_PRIMITIVE: - check_primitive(item); - break; - case IDTYPE_POINTER: - check_pointer(item); - break; - case IDTYPE_CONTAINER: - case IDTYPE_PTR_CONTAINER: - case IDTYPE_BIT_CONTAINER: - case IDTYPE_STL_PTR_VECTOR: - check_container(item); - break; - case IDTYPE_BUFFER: - { - auto item_identity = static_cast(item.identity)->getItemType(); - auto ienum = static_cast(static_cast(item.identity)->getIndexEnumType()); - queue_static_array(item, item.ptr, item_identity, item.identity->byte_size() / item_identity->byte_size(), false, ienum); - } - break; - case IDTYPE_BITFIELD: - check_bitfield(item); - break; - case IDTYPE_ENUM: - check_enum(item); - break; - case IDTYPE_UNION: - FAIL("untagged union"); - check_struct(item); - break; - case IDTYPE_STRUCT: - check_struct(item); - break; - case IDTYPE_CLASS: - check_virtual(item); - break; - case IDTYPE_OPAQUE: - // can't check opaque - break; - } -} - -void Checker::check_global(const ToCheck & globals) -{ - auto identity = static_cast(globals.identity); - - for (auto field = identity->getFields(); field->mode != struct_field_info::END; field++) - { - ToCheck item(globals, field->name, nullptr, field->type); - item.path.push_back(""); // tell check_struct that this is a pointer - - auto base = reinterpret_cast(field->offset); - if (!check_access(item, base, df::identity_traits::get())) - { - continue; - } - - item.ptr = *base; - - if (!seen_addr.insert(item.ptr).second) - { - continue; - } - - queue_field(std::move(item), field); - } -} - -void Checker::check_primitive(const ToCheck & item) -{ - if (item.identity->getFullName() == "string") - { - check_stl_string(item); - return; - } - - if (item.identity->getFullName() == "bool") - { - auto value = *reinterpret_cast(item.ptr); - if (value > 1 && value != 0xd2) - { - FAIL("invalid boolean value " << stl_sprintf("%d (0x%02x)", value, value)); - } - return; - } - - // TODO: check other primitives? -} - -void Checker::check_stl_string(const ToCheck & item) -{ - if (!seen_addr.insert(item.ptr).second) - { - return; - } - - if (!check_access(item, item.ptr, item.identity)) - { - return; - } - -#ifdef WIN32 - struct string_data - { - union - { - uintptr_t start; - char local_data[16]; - }; - size_t length; - size_t capacity; - }; -#else - struct string_data - { - struct string_data_inner - { - size_t length; - size_t capacity; - int32_t refcount; - } *ptr; - }; -#endif - - if (item.identity->byte_size() != sizeof(string_data)) - { - UNEXPECTED; - return; - } - - auto string = reinterpret_cast(item.ptr); -#ifdef WIN32 - bool is_local = string->capacity < 16; - char *start = is_local ? &string->local_data[0] : reinterpret_cast(string->start); - ptrdiff_t length = string->length; - ptrdiff_t capacity = string->capacity; -#else - if (!check_access(item, string->ptr, item.identity, 1)) - { - // nullptr is NOT okay here - FAIL("invalid string pointer " << stl_sprintf("%p", string->ptr)); - return; - } - if (!check_access(item, string->ptr - 1, item.identity, sizeof(*string->ptr))) - { - return; - } - char *start = reinterpret_cast(string->ptr); - ptrdiff_t length = (string->ptr - 1)->length; - ptrdiff_t capacity = (string->ptr - 1)->capacity; -#endif - - if (length < 0) - { - FAIL("string length is negative (" << length << ")"); - } - else if (capacity < 0) - { - FAIL("string capacity is negative (" << capacity << ")"); - } - else if (capacity < length) - { - FAIL("string capacity (" << capacity << ") is less than length (" << length << ")"); - } - -#ifndef WIN32 - const std::string empty_string; - auto empty_string_data = reinterpret_cast(&empty_string); - if (sizes && string->ptr != empty_string_data->ptr) - { - uint32_t tag = *reinterpret_cast(PTR_ADD(string->ptr - 1, -8)); - if (tag == 0xdfdf4ac8) - { - size_t allocated_size = *reinterpret_cast(PTR_ADD(string->ptr - 1, -16)); - size_t expected_size = sizeof(*string->ptr) + capacity + 1; - - if (allocated_size != expected_size) - { - FAIL("allocated string data size (" << allocated_size << ") does not match expected size (" << expected_size << ")"); - } - } - else - { - FAIL("pointer does not appear to be a string"); - //UNEXPECTED; - } - } -#endif - - check_access(item, start, item.identity, capacity); -} - -void Checker::check_pointer(const ToCheck & item) -{ - if (!seen_addr.insert(item.ptr).second) - { - return; - } - - auto base = *reinterpret_cast(item.ptr); - auto base_int = uintptr_t(base); - if (base_int != UNINIT_PTR && base_int % alignof(void *) != 0) - { - FAIL("unaligned pointer " << stl_sprintf("%p", base)); - } - - auto target_identity = static_cast(item.identity)->getTarget(); - queue.push_back(ToCheck(item, "", base, target_identity)); -} - -void Checker::check_bitfield(const ToCheck & item) -{ - if (!enums) - { - return; - } - - auto identity = static_cast(item.identity); - uint64_t val = 0; - for (size_t offset = 0; offset < identity->byte_size(); offset++) - { - val |= uint64_t(*reinterpret_cast(PTR_ADD(item.ptr, offset))) << (8 * offset); - } - - size_t num_bits = identity->getNumBits(); - auto bits = identity->getBits(); - size_t next_bit = 0; - for (size_t i = 0; i < num_bits; i++) - { - if (bits[i].size) - next_bit = i + 1; - - if (bits[i].size < 0) - continue; - if (bits[i].name) - continue; - - if (!(val & (1ULL << i))) - continue; - - if (!bits[i].size) - { - FAIL("bitfield bit " << i << " past the defined end of the bitfield (" << next_bit << " would be next)"); - } - else if (unnamed) - { - FAIL("bitfield bit " << i << " is unnamed"); - } - } -} - -int64_t Checker::check_enum(const ToCheck & item) -{ - auto identity = static_cast(item.identity); - - int64_t value; - switch (identity->byte_size()) - { - case 1: - if (identity->getFirstItem() < 0) - value = *reinterpret_cast(item.ptr); - else - value = *reinterpret_cast(item.ptr); - break; - case 2: - if (identity->getFirstItem() < 0) - value = *reinterpret_cast(item.ptr); - else - value = *reinterpret_cast(item.ptr); - break; - case 4: - if (identity->getFirstItem() < 0) - value = *reinterpret_cast(item.ptr); - else - value = *reinterpret_cast(item.ptr); - break; - case 8: - value = *reinterpret_cast(item.ptr); - break; - default: - UNEXPECTED; - return -1; - } - - if (!enums) - { - return value; - } - - auto key = get_enum_item_key(identity, value); - if (!key) - { - if (identity->getComplex()) - { - FAIL("enum value (" << value << ") is not defined (complex enum)"); - } - else - { - FAIL("enum value (" << value << ") outside of defined range (" << identity->getFirstItem() << " to " << identity->getLastItem() << ")"); - } - } - else if (!*key && unnamed) - { - FAIL("enum value (" << value << ") is unnamed"); - } - - return value; -} - -void Checker::check_container(const ToCheck & item) -{ - auto identity = static_cast(item.identity); - - if (!seen_addr.insert(item.ptr).second) - { - return; - } - - auto void_name = identity->getFullName(nullptr); - if (void_name == "vector") - { - check_vector(item, identity->getItemType(), false); - } - else if (void_name == "vector") - { - check_vector(item, identity->getItemType(), true); - } - else if (void_name == "deque") - { - check_deque(item, identity->getItemType()); - } - else if (void_name == "DfArray") - { - check_dfarray(item, identity->getItemType()); - } - else if (void_name == "BitArray<>") - { - check_bitarray(item); - } - else if (void_name == "vector") - { - check_bitvector(item); - } - else - { - FAIL("TODO: " << void_name); - UNEXPECTED; - } -} - -size_t Checker::check_vector_size(const ToCheck & item, size_t item_size) -{ - struct vector_data - { - uintptr_t start; - uintptr_t finish; - uintptr_t end_of_storage; - }; - - if (item.identity->byte_size() != sizeof(vector_data)) - { - UNEXPECTED; - return 0; - } - - vector_data vector = *reinterpret_cast(item.ptr); - - ptrdiff_t length = vector.finish - vector.start; - ptrdiff_t capacity = vector.end_of_storage - vector.start; - - bool local_ok = true; - if (vector.start > vector.finish) - { - local_ok = false; - FAIL("vector length is negative (" << (length / ptrdiff_t(item_size)) << ")"); - } - if (vector.start > vector.end_of_storage) - { - local_ok = false; - FAIL("vector capacity is negative (" << (capacity / ptrdiff_t(item_size)) << ")"); - } - else if (vector.finish > vector.end_of_storage) - { - local_ok = false; - FAIL("vector capacity (" << (capacity / ptrdiff_t(item_size)) << ") is less than its length (" << (length / ptrdiff_t(item_size)) << ")"); - } - - if (!item_size) - { - return 0; - } - - size_t ulength = size_t(length); - size_t ucapacity = size_t(capacity); - if (ulength % item_size != 0) - { - local_ok = false; - FAIL("vector length is non-integer (" << (ulength / item_size) << " items plus " << (ulength % item_size) << " bytes)"); - } - if (ucapacity % item_size != 0) - { - local_ok = false; - FAIL("vector capacity is non-integer (" << (ucapacity / item_size) << " items plus " << (ucapacity % item_size) << " bytes)"); - } - - if (local_ok && capacity && !vector.start) - { - FAIL("vector has null pointer but capacity " << (capacity / item_size)); - return 0; - } - - if (!check_access(item, reinterpret_cast(vector.start), item.identity, capacity)) - { - return 0; - } - - return local_ok ? ulength / item_size : 0; -} - -void Checker::check_vector(const ToCheck & item, type_identity *item_identity, bool pointer) -{ - size_t item_size = pointer ? sizeof(void *) : item_identity->byte_size(); - - if (!item_identity && pointer && !sizes) - { - // non-identified vector type in structures - item_size = 0; - } - - size_t count = check_vector_size(item, item_size); - - if (item.path.back() == ".bad" || count == 0) - { - // don't check contents - return; - } - - void *start = *reinterpret_cast(item.ptr); - auto ienum = static_cast(static_cast(item.identity)->getIndexEnumType()); - queue_static_array(item, start, item_identity, count, pointer, ienum); -} - -void Checker::check_deque(const ToCheck & item, type_identity *item_identity) -{ - // TODO: check deque? -} - -void Checker::check_dfarray(const ToCheck & item, type_identity *item_identity) -{ - struct dfarray_data - { - uintptr_t start; - unsigned short size; - }; - - if (item.identity->byte_size() != sizeof(dfarray_data)) - { - UNEXPECTED; - return; - } - - dfarray_data dfarray = *reinterpret_cast(item.ptr); - - size_t length = dfarray.size; - size_t item_size = item_identity->byte_size(); - - if (check_access(item, reinterpret_cast(dfarray.start), item.identity, item_size * length)) - { - auto ienum = static_cast(static_cast(item.identity)->getIndexEnumType()); - queue_static_array(item, reinterpret_cast(dfarray.start), item_identity, length, false, ienum); - } -} - -void Checker::check_bitarray(const ToCheck & item) -{ - // TODO: check DFHack::BitArray? -} - -bool Checker::check_bitvector(const ToCheck & item) -{ - struct biterator_data - { - uintptr_t ptr; - unsigned int offset; - }; - - struct bvector_data - { - biterator_data start; - biterator_data finish; - uintptr_t end_of_storage; - }; - - if (item.identity->byte_size() != sizeof(bvector_data)) - { - UNEXPECTED; - return false; - } - - // TODO: check vector? - return true; -} - -void Checker::check_struct(const ToCheck & item) -{ - bool is_pointer = item.path.back().empty(); - bool is_virtual = !item.path.back().empty() && item.path.back().at(0) == '<'; - bool is_virtual_pointer = is_virtual && item.path.size() >= 2 && item.path.at(item.path.size() - 2).empty(); - if (sizes && uintptr_t(item.ptr) % 32 == 16 && (is_pointer || is_virtual_pointer)) - { - uint32_t tag = *reinterpret_cast(PTR_ADD(item.ptr, -8)); - if (tag == 0xdfdf4ac8) - { - size_t allocated_size = *reinterpret_cast(PTR_ADD(item.ptr, -16)); - size_t expected_size = item.identity->byte_size(); - - if (allocated_size != expected_size) - { - FAIL("allocated structure size (" << allocated_size << ") does not match expected size (" << expected_size << ")"); - - if (allocated_size >= MIN_SIZE_FOR_SUGGEST && known_types_by_size.count(allocated_size)) - { - FAIL("known types of this size: " << join_strings(", ", known_types_by_size.at(allocated_size))); - } - } - } - else - { - FAIL("unknown allocation size; possibly bad"); - //UNEXPECTED; - } - } - - for (auto identity = static_cast(item.identity); identity; identity = identity->getParent()) - { - auto fields = identity->getFields(); - if (!fields) - { - continue; - } - - for (auto field = fields; field->mode != struct_field_info::END; field++) - { - if (maybe_queue_union(item, fields, field)) - { - continue; - } - - if (field->mode == struct_field_info::POINTER && is_df_linked_list(field->type)) - { - // skip linked list pointers - continue; - } - - ToCheck child(item, std::string(".") + field->name, PTR_ADD(item.ptr, field->offset), field->type); - - queue_field(std::move(child), field); - } - } -} - -void Checker::check_virtual(const ToCheck & item) -{ - if (!seen_addr.insert(item.ptr).second) - { - return; - } - - if (!check_access(item, item.ptr, item.identity)) - { - return; - } - - auto identity = static_cast(item.identity); - - void *vtable = *reinterpret_cast(item.ptr); - if (!check_vtable(item, vtable, identity)) - { - FAIL("invalid vtable pointer"); - return; - } - else if (!identity->is_instance(reinterpret_cast(item.ptr))) - { - auto class_name = Core::getInstance().p->readClassName(vtable); - FAIL("vtable is not a known subclass (subclass is " << class_name << ")"); - return; - } - - auto vident = virtual_identity::get(reinterpret_cast(item.ptr)); - ToCheck virtual_item(item, "<" + vident->getFullName() + ">", item.ptr, vident); - check_struct(virtual_item); -} diff --git a/plugins/devel/check-structures-sanity/CMakeLists.txt b/plugins/devel/check-structures-sanity/CMakeLists.txt new file mode 100644 index 000000000..f3eab5da6 --- /dev/null +++ b/plugins/devel/check-structures-sanity/CMakeLists.txt @@ -0,0 +1,8 @@ +set(PLUGIN_SRCS + dispatch.cpp + main.cpp + types.cpp + validate.cpp +) + +dfhack_plugin(check-structures-sanity ${PLUGIN_SRCS} LINK_LIBRARIES lua) diff --git a/plugins/devel/check-structures-sanity/check-structures-sanity.h b/plugins/devel/check-structures-sanity/check-structures-sanity.h new file mode 100644 index 000000000..847f59ae9 --- /dev/null +++ b/plugins/devel/check-structures-sanity/check-structures-sanity.h @@ -0,0 +1,117 @@ +#pragma once + +#include "Console.h" +#include "PluginManager.h" +#include "MemAccess.h" +#include "DataDefs.h" +#include "DataIdentity.h" + +#include +#include +#include + +using namespace DFHack; + +#ifdef WIN32 +#define UNEXPECTED __debugbreak() +#else +#define UNEXPECTED __asm__ volatile ("int $0x03") +#endif + +#define PTR_ADD(ptr, offset) reinterpret_cast(uintptr_t(ptr) + (offset)) + +struct QueueItem +{ + QueueItem(const std::string &, const void *); + QueueItem(const QueueItem &, const std::string &, const void *); + QueueItem(const QueueItem &, size_t, const void *); + + std::string path; + const void *ptr; +}; +struct CheckedStructure +{ + type_identity *identity; + size_t count; + + CheckedStructure(); + explicit CheckedStructure(type_identity *, size_t = 0); + CheckedStructure(const struct_field_info *); + + size_t full_size() const; +}; + +namespace +{ + template::value> + struct safe_t + { + typedef T type; + }; + template + struct safe_t + { + typedef void *type; + }; +} + +class Checker +{ + color_ostream & out; + std::vector mapped; + std::map data; + std::deque queue; +public: + size_t checked_count; + size_t error_count; + size_t maxerrors; + bool maxerrors_reported; + bool enums; + bool sizes; + bool unnamed; + bool failfast; + + Checker(color_ostream & out); + void queue_item(const QueueItem & item, const CheckedStructure & cs); + void queue_globals(); + bool process_queue(); + + bool is_valid_dereference(const QueueItem & item, const CheckedStructure & cs, bool quiet = false); + template + const T validate_and_dereference(const QueueItem & item, bool quiet = false) + { + CheckedStructure cs; + cs.identity = df::identity_traits::type>::get(); + if (!is_valid_dereference(item, cs, quiet)) + return T(); + + return *reinterpret_cast(item.ptr); + } + const char *get_vtable_name(const QueueItem & item, const CheckedStructure & cs, bool quiet = false); + std::pair validate_vector_size(const QueueItem & item, const CheckedStructure & cs, bool quiet = false); + +private: + color_ostream & fail(int, const QueueItem &, const CheckedStructure &); + void dispatch_item(const QueueItem &, const CheckedStructure &); + void dispatch_single_item(const QueueItem &, const CheckedStructure &); + void dispatch_primitive(const QueueItem &, const CheckedStructure &); + void dispatch_pointer(const QueueItem &, const CheckedStructure &); + void dispatch_container(const QueueItem &, const CheckedStructure &); + void dispatch_ptr_container(const QueueItem &, const CheckedStructure &); + void dispatch_bit_container(const QueueItem &, const CheckedStructure &); + void dispatch_bitfield(const QueueItem &, const CheckedStructure &); + void dispatch_enum(const QueueItem &, const CheckedStructure &); + void dispatch_struct(const QueueItem &, const CheckedStructure &); + void dispatch_field(const QueueItem &, const CheckedStructure &, const struct_field_info *, const struct_field_info *); + void dispatch_class(const QueueItem &, const CheckedStructure &); + void dispatch_buffer(const QueueItem &, const CheckedStructure &); + void dispatch_stl_ptr_vector(const QueueItem &, const CheckedStructure &); + void dispatch_untagged_union(const QueueItem &, const CheckedStructure &); + void check_stl_vector(const QueueItem &, type_identity *); + + friend struct CheckedStructure; + static type_identity *wrap_in_pointer(type_identity *); + static type_identity *wrap_in_stl_ptr_vector(type_identity *); +}; + +#define FAIL(message) (static_cast(fail(__LINE__, item, cs) << message) << COLOR_RESET << std::endl) diff --git a/plugins/devel/check-structures-sanity/dispatch.cpp b/plugins/devel/check-structures-sanity/dispatch.cpp new file mode 100644 index 000000000..18600a58d --- /dev/null +++ b/plugins/devel/check-structures-sanity/dispatch.cpp @@ -0,0 +1,389 @@ +#include "check-structures-sanity.h" + +#include "df/large_integer.h" + +Checker::Checker(color_ostream & out) : + out(out), + checked_count(0), + error_count(0), + maxerrors(~size_t(0)), + maxerrors_reported(false), + enums(false), + sizes(false), + unnamed(false), + failfast(false) +{ + Core::getInstance().p->getMemRanges(mapped); +} + +color_ostream & Checker::fail(int line, const QueueItem & item, const CheckedStructure & cs) +{ + error_count++; + out << COLOR_LIGHTRED << "sanity check failed (line " << line << "): "; + out << COLOR_RESET << (cs.identity ? cs.identity->getFullName() : "?"); + out << " (accessed as " << item.path << "): "; + out << COLOR_YELLOW; + if (maxerrors && maxerrors != ~size_t(0)) + maxerrors--; + if (failfast) + UNEXPECTED; + return out; +} + +void Checker::queue_item(const QueueItem & item, const CheckedStructure & cs) +{ + if (data.count(item.ptr) && data.at(item.ptr).full_size() == cs.full_size()) + { + // already checked + // TODO: make sure types are equal + UNEXPECTED; + return; + } + + auto ptr_end = PTR_ADD(item.ptr, cs.full_size()); + + auto prev = data.lower_bound(item.ptr); + if (prev != data.cbegin()) + { + prev--; + if (uintptr_t(prev->first) + prev->second.full_size() > uintptr_t(item.ptr)) + { + // TODO + UNEXPECTED; + } + } + + auto overlap = data.lower_bound(item.ptr); + auto overlap_end = data.lower_bound(ptr_end); + while (overlap != overlap_end) + { + // TODO + UNEXPECTED; + overlap++; + } + + data[item.ptr] = cs; + queue.push_back(item); +} + +void Checker::queue_globals() +{ + auto fields = df::global::_identity.getFields(); + for (auto field = fields; field->mode != struct_field_info::END; field++) + { + if (!field->offset) + { + UNEXPECTED; + continue; + } + + // offset is the position of the DFHack pointer to this global. + auto ptr = *reinterpret_cast(field->offset); + + QueueItem item(stl_sprintf("df.global.%s", field->name), ptr); + CheckedStructure cs(field); + + if (!ptr) + { + FAIL("unknown global address"); + continue; + } + + queue_item(item, cs); + } +} + +bool Checker::process_queue() +{ + if (queue.empty()) + { + return false; + } + + auto item = std::move(queue.front()); + queue.pop_front(); + + auto cs = data.find(item.ptr); + if (cs == data.end()) + { + // happens if pointer is determined to be part of a larger structure + return true; + } + + dispatch_item(item, cs->second); + + return true; +} + + +void Checker::dispatch_item(const QueueItem & base, const CheckedStructure & cs) +{ + if (!is_valid_dereference(base, cs)) + { + return; + } + + if (!cs.count) + { + dispatch_single_item(base, cs); + return; + } + + auto ptr = base.ptr; + auto size = cs.identity->byte_size(); + for (size_t i = 0; i < cs.count; i++) + { + QueueItem item(base, i, ptr); + dispatch_single_item(item, cs); + ptr = PTR_ADD(ptr, size); + } +} + +void Checker::dispatch_single_item(const QueueItem & item, const CheckedStructure & cs) +{ + checked_count++; + + if (!maxerrors) + { + if (!maxerrors_reported) + { + FAIL("error limit reached. bailing out with " << (queue.size() + 1) << " items remaining in the queue."); + maxerrors_reported = true; + } + queue.clear(); + return; + } + + switch (cs.identity->type()) + { + case IDTYPE_GLOBAL: + case IDTYPE_FUNCTION: + UNEXPECTED; + break; + case IDTYPE_PRIMITIVE: + dispatch_primitive(item, cs); + break; + case IDTYPE_POINTER: + dispatch_pointer(item, cs); + break; + case IDTYPE_CONTAINER: + dispatch_container(item, cs); + break; + case IDTYPE_PTR_CONTAINER: + dispatch_ptr_container(item, cs); + break; + case IDTYPE_BIT_CONTAINER: + dispatch_bit_container(item, cs); + break; + case IDTYPE_BITFIELD: + dispatch_bitfield(item, cs); + break; + case IDTYPE_ENUM: + dispatch_enum(item, cs); + break; + case IDTYPE_STRUCT: + dispatch_struct(item, cs); + break; + case IDTYPE_CLASS: + dispatch_class(item, cs); + break; + case IDTYPE_BUFFER: + dispatch_buffer(item, cs); + break; + case IDTYPE_STL_PTR_VECTOR: + dispatch_stl_ptr_vector(item, cs); + break; + case IDTYPE_OPAQUE: + UNEXPECTED; + break; + case IDTYPE_UNION: + dispatch_untagged_union(item, cs); + break; + } +} + +void Checker::dispatch_primitive(const QueueItem & item, const CheckedStructure & cs) +{ + if (cs.identity->isConstructed()) + { + if (cs.identity == df::identity_traits::get()) + { + // TODO check std::string + UNEXPECTED; + } + else + { + UNEXPECTED; + } + } + + // TODO: check primitives + UNEXPECTED; +} +void Checker::dispatch_pointer(const QueueItem & item, const CheckedStructure & cs) +{ + auto identity = static_cast(cs.identity); + // TODO: check pointer + UNEXPECTED; +} +void Checker::dispatch_container(const QueueItem & item, const CheckedStructure & cs) +{ + auto identity = static_cast(cs.identity); + auto base_container = identity->getFullName(nullptr); + if (base_container == "vector") + { + if (identity->getIndexEnumType()) + { + UNEXPECTED; + } + check_stl_vector(item, identity->getItemType()); + } + else if (base_container == "deque") + { + // TODO: check deque? + } + else + { + UNEXPECTED; + } +} +void Checker::dispatch_ptr_container(const QueueItem & item, const CheckedStructure & cs) +{ + auto identity = static_cast(cs.identity); + auto base_container = identity->getFullName(nullptr); + { + UNEXPECTED; + } +} +void Checker::dispatch_bit_container(const QueueItem & item, const CheckedStructure & cs) +{ + auto identity = static_cast(cs.identity); + auto base_container = identity->getFullName(nullptr); + if (base_container == "BitArray<>") + { + // TODO: check DF bit array + UNEXPECTED; + } + else if (base_container == "vector") + { + // TODO: check stl bit vector + UNEXPECTED; + } + else + { + UNEXPECTED; + } +} +void Checker::dispatch_bitfield(const QueueItem & item, const CheckedStructure & cs) +{ + if (!enums) + { + return; + } + + // TODO: check bitfields + UNEXPECTED; +} +void Checker::dispatch_enum(const QueueItem & item, const CheckedStructure & cs) +{ + if (!enums) + { + return; + } + + // TODO: check enums + UNEXPECTED; +} +void Checker::dispatch_struct(const QueueItem & item, const CheckedStructure & cs) +{ + auto identity = static_cast(cs.identity); + for (auto p = identity; p; p = p->getParent()) + { + auto fields = p->getFields(); + for (auto field = fields; field->mode != struct_field_info::END; field++) + { + dispatch_field(item, cs, fields, field); + } + } +} +void Checker::dispatch_field(const QueueItem & item, const CheckedStructure & cs, const struct_field_info *fields, const struct_field_info *field) +{ + if (field->mode == struct_field_info::OBJ_METHOD || + field->mode == struct_field_info::CLASS_METHOD) + { + return; + } + + auto tag_field = find_union_tag(fields, field); + if (tag_field) + { + UNEXPECTED; + return; + } + + auto field_ptr = PTR_ADD(item.ptr, field->offset); + CheckedStructure field_cs(field); + dispatch_item(QueueItem(item, field->name, field_ptr), field_cs); +} +void Checker::dispatch_class(const QueueItem & item, const CheckedStructure & cs) +{ + auto vtable_name = get_vtable_name(item, cs); + if (!vtable_name) + { + // bail out now because virtual_identity::get will crash + return; + } + + auto base_identity = static_cast(cs.identity); + auto vptr = static_cast(const_cast(item.ptr)); + if (!base_identity->is_instance(vptr)) + { + FAIL("expected subclass of " << base_identity->getFullName() << ", but got " << vtable_name); + return; + } + + auto identity = virtual_identity::get(vptr); + dispatch_struct(QueueItem(item.path + "<" + identity->getFullName() + ">", item.ptr), CheckedStructure(identity)); +} +void Checker::dispatch_buffer(const QueueItem & item, const CheckedStructure & cs) +{ + auto identity = static_cast(cs.identity); + if (identity->getIndexEnumType()) + { + UNEXPECTED; + } + + auto item_identity = identity->getItemType(); + dispatch_item(item, CheckedStructure(item_identity, identity->byte_size() / item_identity->byte_size())); +} +void Checker::dispatch_stl_ptr_vector(const QueueItem & item, const CheckedStructure & cs) +{ + auto identity = static_cast(cs.identity); + if (identity->getIndexEnumType()) + { + UNEXPECTED; + } + auto ptr_type = wrap_in_pointer(identity->getItemType()); + check_stl_vector(item, ptr_type); +} +void Checker::dispatch_untagged_union(const QueueItem & item, const CheckedStructure & cs) +{ + if (cs.identity == df::identity_traits::get()) + { + dispatch_primitive(item, CheckedStructure(df::identity_traits::get())); + return; + } + + UNEXPECTED; +} + +void Checker::check_stl_vector(const QueueItem & item, type_identity *item_identity) +{ + auto vec_items = validate_vector_size(item, CheckedStructure(item_identity)); + if (vec_items.first && vec_items.second) + { + QueueItem items_item(item.path, vec_items.first); + CheckedStructure items_cs(item_identity, vec_items.second); + queue_item(items_item, items_cs); + } +} diff --git a/plugins/devel/check-structures-sanity/main.cpp b/plugins/devel/check-structures-sanity/main.cpp new file mode 100644 index 000000000..055cbe765 --- /dev/null +++ b/plugins/devel/check-structures-sanity/main.cpp @@ -0,0 +1,126 @@ +#include "check-structures-sanity.h" + +#include "LuaTools.h" +#include "LuaWrapper.h" + +DFHACK_PLUGIN("check-structures-sanity"); + +static command_result command(color_ostream &, std::vector &); + +DFhackCExport command_result plugin_init(color_ostream &, std::vector & commands) +{ + commands.push_back(PluginCommand( + "check-structures-sanity", + "performs a sanity check on df-structures", + command, + false, + "check-structures-sanity [-enums] [-sizes] [-lowmem] [-maxerrors n] [-failfast] [starting_point]\n" + "\n" + "-enums: report unexpected or unnamed enum or bitfield values.\n" + "-sizes: report struct and class sizes that don't match structures. (requires sizecheck)\n" + "-unnamed: report unnamed enum/bitfield values, not just undefined ones.\n" + "-maxerrors n: set the maximum number of errors before bailing out.\n" + "-failfast: crash if any error is encountered. useful only for debugging.\n" + "starting_point: a lua expression or a word like 'screen', 'item', or 'building'. (defaults to df.global)\n" + "\n" + "by default, check-structures-sanity reports invalid pointers, vectors, strings, and vtables." + )); + + return CR_OK; +} + +static command_result command(color_ostream & out, std::vector & parameters) +{ + CoreSuspender suspend; + + Checker checker(out); + // check parameters with values first +#define VAL_PARAM(name, expr_using_value) \ + auto name ## _idx = std::find(parameters.begin(), parameters.end(), "-" #name); \ + if (name ## _idx != parameters.end()) \ + { \ + if (name ## _idx + 1 == parameters.end()) \ + { \ + return CR_WRONG_USAGE; \ + } \ + try \ + { \ + auto value = std::move(*(name ## _idx + 1)); \ + parameters.erase((name ## _idx + 1)); \ + parameters.erase(name ## _idx); \ + checker.name = (expr_using_value); \ + } \ + catch (std::exception & ex) \ + { \ + out.printerr("check-structures-sanity: argument to -%s: %s\n", #name, ex.what()); \ + return CR_WRONG_USAGE; \ + } \ + } + VAL_PARAM(maxerrors, std::stoul(value)); +#undef VAL_PARAM + +#define BOOL_PARAM(name) \ + auto name ## _idx = std::find(parameters.begin(), parameters.end(), "-" #name); \ + if (name ## _idx != parameters.end()) \ + { \ + checker.name = true; \ + parameters.erase(name ## _idx); \ + } + BOOL_PARAM(enums); + BOOL_PARAM(sizes); + BOOL_PARAM(unnamed); + BOOL_PARAM(failfast); +#undef BOOL_PARAM + + if (parameters.size() > 1) + { + return CR_WRONG_USAGE; + } + + if (parameters.empty()) + { + checker.queue_globals(); + } + else + { + using namespace DFHack::Lua; + using namespace DFHack::Lua::Core; + using namespace DFHack::LuaWrapper; + + StackUnwinder unwinder(State); + PushModulePublic(out, "utils", "df_expr_to_ref"); + Push(parameters.at(0)); + if (!SafeCall(out, 1, 1)) + { + return CR_FAILURE; + } + if (!lua_touserdata(State, -1)) + { + return CR_WRONG_USAGE; + } + + QueueItem item(parameters.at(0), get_object_ref(State, -1)); + lua_getfield(State, -1, "_type"); + lua_getfield(State, -1, "_identity"); + auto identity = reinterpret_cast(lua_touserdata(State, -1)); + if (!identity) + { + out.printerr("could not determine type identity\n"); + return CR_FAILURE; + } + + checker.queue_item(item, CheckedStructure(identity)); + } + + while (checker.process_queue()) + { + if (out.is_console()) + { + out << "checked " << checker.checked_count << " fields\r" << std::flush; + } + } + + out << "checked " << checker.checked_count << " fields" << std::endl; + + return checker.error_count ? CR_FAILURE : CR_OK; +} diff --git a/plugins/devel/check-structures-sanity/types.cpp b/plugins/devel/check-structures-sanity/types.cpp new file mode 100644 index 000000000..9ee880b7a --- /dev/null +++ b/plugins/devel/check-structures-sanity/types.cpp @@ -0,0 +1,131 @@ +#include "check-structures-sanity.h" + +QueueItem::QueueItem(const std::string & path, const void *ptr) : + path(path), + ptr(ptr) +{ +} +QueueItem::QueueItem(const QueueItem & parent, const std::string & member, const void *ptr) : + QueueItem(parent.path + "." + member, ptr) +{ +} +QueueItem::QueueItem(const QueueItem & parent, size_t index, const void *ptr) : + QueueItem(parent.path + stl_sprintf("[%zu]", index), ptr) +{ +} + +CheckedStructure::CheckedStructure() : + CheckedStructure(nullptr, 0) +{ +} +CheckedStructure::CheckedStructure(type_identity *identity, size_t count) : + identity(identity), + count(count) +{ +} +CheckedStructure::CheckedStructure(const struct_field_info *field) : + CheckedStructure() +{ + if (!field || field->mode == struct_field_info::END) + { + UNEXPECTED; + } + + if (field->mode == struct_field_info::STATIC_ARRAY && field->eid) + { + UNEXPECTED; + } + else if (field->type && field->type->isContainer()) + { + auto expected_eid = static_cast(field->type)->getIndexEnumType(); + if (field->eid != expected_eid) + { + UNEXPECTED; + } + } + else if (field->eid) + { + UNEXPECTED; + } + + identity = field->type; + switch (field->mode) + { + case struct_field_info::END: + UNEXPECTED; + break; + case struct_field_info::PRIMITIVE: + if (field->count || !field->type) + { + UNEXPECTED; + } + break; + case struct_field_info::STATIC_STRING: + if (!field->count || field->type) + { + UNEXPECTED; + } + identity = df::identity_traits::get(); + count = field->count; + break; + case struct_field_info::POINTER: + // TODO: check flags (stored in field->count) + identity = Checker::wrap_in_pointer(field->type); + break; + case struct_field_info::STATIC_ARRAY: + if (!field->count || !field->type) + { + UNEXPECTED; + } + count = field->count; + break; + case struct_field_info::SUBSTRUCT: + case struct_field_info::CONTAINER: + if (field->count || !field->type) + { + UNEXPECTED; + } + break; + case struct_field_info::STL_VECTOR_PTR: + if (field->count) + { + UNEXPECTED; + } + identity = Checker::wrap_in_stl_ptr_vector(field->type); + break; + case struct_field_info::OBJ_METHOD: + case struct_field_info::CLASS_METHOD: + UNEXPECTED; + break; + } +} + +size_t CheckedStructure::full_size() const +{ + size_t size = identity->byte_size(); + if (count) + { + size *= count; + } + + return size; +} + +#define RETURN_CACHED_WRAPPER(T, base, ...) \ + static std::map> wrappers; \ + auto it = wrappers.find(base); \ + if (it != wrappers.end()) \ + { \ + return it->second.get(); \ + } \ + return (wrappers[base] = dts::make_unique(base __VA_OPT__(,) __VA_ARGS__)).get() + +type_identity *Checker::wrap_in_stl_ptr_vector(type_identity *base) +{ + RETURN_CACHED_WRAPPER(df::stl_ptr_vector_identity, base, nullptr); +} + +type_identity *Checker::wrap_in_pointer(type_identity *base) +{ + RETURN_CACHED_WRAPPER(df::pointer_identity, base); +} diff --git a/plugins/devel/check-structures-sanity/validate.cpp b/plugins/devel/check-structures-sanity/validate.cpp new file mode 100644 index 000000000..6bae2e1bc --- /dev/null +++ b/plugins/devel/check-structures-sanity/validate.cpp @@ -0,0 +1,237 @@ +#include "check-structures-sanity.h" + +bool Checker::is_valid_dereference(const QueueItem & item, const CheckedStructure & cs, bool quiet) +{ + auto base = const_cast(item.ptr); + auto size = cs.full_size(); + if (!base) + { + // cannot dereference null pointer, but not an error + return false; + } + + // assumes MALLOC_PERTURB_=45 +#ifdef DFHACK64 +#define UNINIT_PTR 0xd2d2d2d2d2d2d2d2 +#define FAIL_PTR(message) FAIL(stl_sprintf("0x%016zx: ", reinterpret_cast(base)) << message) +#else +#define UNINIT_PTR 0xd2d2d2d2 +#define FAIL_PTR(message) FAIL(stl_sprintf("0x%08zx: ", reinterpret_cast(base)) << message) +#endif + if (uintptr_t(base) == UNINIT_PTR) + { + if (!quiet) + { + FAIL_PTR("uninitialized pointer"); + } + return false; + } + + bool found = true; + auto expected_start = base; + size_t remaining_size = size; + while (found) + { + found = false; + + for (auto & range : mapped) + { + if (!range.isInRange(expected_start)) + { + continue; + } + + found = true; + + if (!range.valid || !range.read) + { + if (!quiet) + { + FAIL_PTR("pointer to invalid memory range"); + } + return false; + } + + auto expected_end = const_cast(PTR_ADD(expected_start, remaining_size - 1)); + if (size && !range.isInRange(expected_end)) + { + auto next_start = PTR_ADD(range.end, 1); + remaining_size -= ptrdiff_t(next_start) - ptrdiff_t(expected_start); + expected_start = const_cast(next_start); + break; + } + + return true; + } + } + + if (quiet) + { + return false; + } + + if (expected_start == base) + { + FAIL_PTR("pointer not in any mapped range"); + } + else + { + FAIL_PTR("pointer exceeds mapped memory bounds (size " << size << ")"); + } + + return false; +#undef FAIL_PTR +} + +const char *Checker::get_vtable_name(const QueueItem & item, const CheckedStructure & cs, bool quiet) +{ + auto vtable = validate_and_dereference(QueueItem(item, "?vtable?", item.ptr), quiet); + if (!vtable) + return nullptr; + + auto info = validate_and_dereference(QueueItem(item, "?vtable?.info", vtable - 1), quiet); + if (!info) + return nullptr; + +#ifdef WIN32 +#ifdef DFHACK64 + void *base; + if (!RtlPcToFileHeader(info, &base)) + return nullptr; + + const char *typeinfo = reinterpret_cast(base) + reinterpret_cast(info)[3]; + const char *name = typeinfo + 16; +#else + const char *name = reinterpret_cast(info) + 8; +#endif +#else + auto name = validate_and_dereference(QueueItem(item, "?vtable?.info.name", info + 1), quiet); +#endif + + for (auto & range : mapped) + { + if (!range.isInRange(const_cast(name))) + { + continue; + } + + if (!range.valid || !range.read) + { + if (!quiet) + { + FAIL("pointer to invalid memory range"); + } + return nullptr; + } + + const char *first_letter = nullptr; + bool letter = false; + for (const char *p = name; ; p++) + { + if (!range.isInRange(const_cast(p))) + { + return nullptr; + } + + if ((*p >= 'a' && *p <= 'z') || *p == '_') + { + if (!letter) + { + first_letter = p; + } + letter = true; + } + else if (!*p) + { + return first_letter; + } + } + } + + return nullptr; +} + +std::pair Checker::validate_vector_size(const QueueItem & item, const CheckedStructure & cs, bool quiet) +{ + struct vector_data + { + uintptr_t start; + uintptr_t finish; + uintptr_t end_of_storage; + }; + + vector_data vector = *reinterpret_cast(item.ptr); + + ptrdiff_t length = vector.finish - vector.start; + ptrdiff_t capacity = vector.end_of_storage - vector.start; + + bool local_ok = true; + auto item_size = cs.full_size(); + if (!item_size) + { + item_size = 1; + local_ok = false; + } + + if (vector.start > vector.finish) + { + local_ok = false; + if (!quiet) + { + FAIL("vector length is negative (" << (length / ptrdiff_t(item_size)) << ")"); + } + } + if (vector.start > vector.end_of_storage) + { + local_ok = false; + if (!quiet) + { + FAIL("vector capacity is negative (" << (capacity / ptrdiff_t(item_size)) << ")"); + } + } + else if (vector.finish > vector.end_of_storage) + { + local_ok = false; + if (!quiet) + { + FAIL("vector capacity (" << (capacity / ptrdiff_t(item_size)) << ") is less than its length (" << (length / ptrdiff_t(item_size)) << ")"); + } + } + + size_t ulength = size_t(length); + size_t ucapacity = size_t(capacity); + if (ulength % item_size != 0) + { + local_ok = false; + if (!quiet) + { + FAIL("vector length is non-integer (" << (ulength / item_size) << " items plus " << (ulength % item_size) << " bytes)"); + } + } + if (ucapacity % item_size != 0) + { + local_ok = false; + if (!quiet) + { + FAIL("vector capacity is non-integer (" << (ucapacity / item_size) << " items plus " << (ucapacity % item_size) << " bytes)"); + } + } + + if (local_ok && capacity && !vector.start) + { + if (!quiet) + { + FAIL("vector has null pointer but capacity " << (capacity / item_size)); + } + return std::make_pair(nullptr, 0); + } + + auto start_ptr = reinterpret_cast(vector.start); + + if (capacity && !is_valid_dereference(QueueItem(item, "?items?", start_ptr), CheckedStructure(cs.identity, capacity / item_size), quiet)) + { + local_ok = false; + } + + return local_ok ? std::make_pair(start_ptr, ulength / item_size) : std::make_pair(nullptr, 0); +} From e2138a6cc2f61927c666a176a4915adfd802de0d Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Tue, 10 Mar 2020 23:05:59 -0500 Subject: [PATCH 20/32] update check-structures-sanity (part 2 of 2) --- .../check-structures-sanity.h | 46 +- .../check-structures-sanity/dispatch.cpp | 528 ++++++++++++++++-- .../devel/check-structures-sanity/main.cpp | 3 + .../devel/check-structures-sanity/types.cpp | 39 +- .../check-structures-sanity/validate.cpp | 139 ++++- 5 files changed, 674 insertions(+), 81 deletions(-) diff --git a/plugins/devel/check-structures-sanity/check-structures-sanity.h b/plugins/devel/check-structures-sanity/check-structures-sanity.h index 847f59ae9..6946faa3f 100644 --- a/plugins/devel/check-structures-sanity/check-structures-sanity.h +++ b/plugins/devel/check-structures-sanity/check-structures-sanity.h @@ -15,7 +15,7 @@ using namespace DFHack; #ifdef WIN32 #define UNEXPECTED __debugbreak() #else -#define UNEXPECTED __asm__ volatile ("int $0x03") +#define UNEXPECTED __asm__ volatile ("int $0x03; nop") #endif #define PTR_ADD(ptr, offset) reinterpret_cast(uintptr_t(ptr) + (offset)) @@ -33,14 +33,20 @@ struct CheckedStructure { type_identity *identity; size_t count; + enum_identity *eid; CheckedStructure(); explicit CheckedStructure(type_identity *, size_t = 0); + CheckedStructure(type_identity *, size_t, enum_identity *); CheckedStructure(const struct_field_info *); size_t full_size() const; }; +#define MIN_SIZE_FOR_SUGGEST 64 +extern std::map> known_types_by_size; +void build_size_table(); + namespace { template::value> @@ -59,7 +65,7 @@ class Checker { color_ostream & out; std::vector mapped; - std::map data; + std::map> data; std::deque queue; public: size_t checked_count; @@ -72,11 +78,19 @@ public: bool failfast; Checker(color_ostream & out); - void queue_item(const QueueItem & item, const CheckedStructure & cs); + bool queue_item(const QueueItem & item, const CheckedStructure & cs); void queue_globals(); bool process_queue(); - bool is_valid_dereference(const QueueItem & item, const CheckedStructure & cs, bool quiet = false); + bool is_valid_dereference(const QueueItem & item, const CheckedStructure & cs, size_t size, bool quiet); + inline bool is_valid_dereference(const QueueItem & item, const CheckedStructure & cs, bool quiet = false) + { + return is_valid_dereference(item, cs, cs.full_size(), quiet); + } + inline bool is_valid_dereference(const QueueItem & item, size_t size, bool quiet = false) + { + return is_valid_dereference(item, CheckedStructure(df::identity_traits::get()), size, quiet); + } template const T validate_and_dereference(const QueueItem & item, bool quiet = false) { @@ -87,8 +101,15 @@ public: return *reinterpret_cast(item.ptr); } + int64_t get_int_value(const QueueItem & item, type_identity *type, bool quiet = false); const char *get_vtable_name(const QueueItem & item, const CheckedStructure & cs, bool quiet = false); std::pair validate_vector_size(const QueueItem & item, const CheckedStructure & cs, bool quiet = false); + size_t get_allocated_size(const QueueItem & item); +#ifndef WIN32 + // this function doesn't make sense on windows, where std::string is not pointer-sized. + const std::string *validate_stl_string_pointer(const void *const*); +#endif + static const char *const *get_enum_item_key(enum_identity *identity, int64_t value); private: color_ostream & fail(int, const QueueItem &, const CheckedStructure &); @@ -106,12 +127,25 @@ private: void dispatch_class(const QueueItem &, const CheckedStructure &); void dispatch_buffer(const QueueItem &, const CheckedStructure &); void dispatch_stl_ptr_vector(const QueueItem &, const CheckedStructure &); + void dispatch_tagged_union(const QueueItem &, const QueueItem &, const CheckedStructure &, const CheckedStructure &); + void dispatch_tagged_union_vector(const QueueItem &, const QueueItem &, const CheckedStructure &, const CheckedStructure &); void dispatch_untagged_union(const QueueItem &, const CheckedStructure &); - void check_stl_vector(const QueueItem &, type_identity *); + void check_unknown_pointer(const QueueItem &); + void check_stl_vector(const QueueItem &, type_identity *, type_identity *); + void check_stl_string(const QueueItem &); friend struct CheckedStructure; static type_identity *wrap_in_pointer(type_identity *); static type_identity *wrap_in_stl_ptr_vector(type_identity *); }; -#define FAIL(message) (static_cast(fail(__LINE__, item, cs) << message) << COLOR_RESET << std::endl) +#define FAIL(message) \ + do \ + { \ + auto & failstream = fail(__LINE__, item, cs); \ + failstream << message; \ + failstream << COLOR_RESET << std::endl; \ + if (failfast) \ + UNEXPECTED; \ + } \ + while (false) diff --git a/plugins/devel/check-structures-sanity/dispatch.cpp b/plugins/devel/check-structures-sanity/dispatch.cpp index 18600a58d..1e02aaf9f 100644 --- a/plugins/devel/check-structures-sanity/dispatch.cpp +++ b/plugins/devel/check-structures-sanity/dispatch.cpp @@ -25,19 +25,25 @@ color_ostream & Checker::fail(int line, const QueueItem & item, const CheckedStr out << COLOR_YELLOW; if (maxerrors && maxerrors != ~size_t(0)) maxerrors--; - if (failfast) - UNEXPECTED; return out; } -void Checker::queue_item(const QueueItem & item, const CheckedStructure & cs) +bool Checker::queue_item(const QueueItem & item, const CheckedStructure & cs) { - if (data.count(item.ptr) && data.at(item.ptr).full_size() == cs.full_size()) + if (data.count(item.ptr)) { // already checked - // TODO: make sure types are equal - UNEXPECTED; - return; + auto existing = data.at(item.ptr); + if (cs.identity != existing.second.identity) + { + if (cs.identity->type() == IDTYPE_CLASS && existing.second.identity->type() == IDTYPE_CLASS && get_vtable_name(item, cs, false) && static_cast(cs.identity)->is_instance(static_cast(const_cast(item.ptr)))) + { + return false; + } + + FAIL("TODO: handle merging structures: " << data.at(item.ptr).first << " overlaps " << item.path << " (same pointer)"); + } + return false; } auto ptr_end = PTR_ADD(item.ptr, cs.full_size()); @@ -46,10 +52,27 @@ void Checker::queue_item(const QueueItem & item, const CheckedStructure & cs) if (prev != data.cbegin()) { prev--; - if (uintptr_t(prev->first) + prev->second.full_size() > uintptr_t(item.ptr)) + if (uintptr_t(prev->first) + prev->second.second.full_size() > uintptr_t(item.ptr)) { + // placeholder algorithm + if (auto sid = dynamic_cast(prev->second.second.identity)) + { + auto offset = uintptr_t(item.ptr) - uintptr_t(prev->first); + for (auto field = sid->getFields(); field->mode != struct_field_info::END; field++) + { + if (field->offset == offset) + { + if (field->mode == struct_field_info::SUBSTRUCT && field->type == cs.identity) + { + return false; + } + + UNEXPECTED; + } + } + } // TODO - UNEXPECTED; + FAIL("TODO: handle merging structures: " << prev->second.first << " overlaps " << item.path << " (backward)"); } } @@ -58,12 +81,13 @@ void Checker::queue_item(const QueueItem & item, const CheckedStructure & cs) while (overlap != overlap_end) { // TODO - UNEXPECTED; + FAIL("TODO: handle merging structures: " << overlap->second.first << " overlaps " << item.path << " (forward)"); overlap++; } - data[item.ptr] = cs; + data[item.ptr] = std::make_pair(item.path, cs); queue.push_back(item); + return true; } void Checker::queue_globals() @@ -89,6 +113,12 @@ void Checker::queue_globals() continue; } + if (!strcmp(field->name, "enabler")) + { + // don't care about libgraphics as we have the source code + continue; + } + queue_item(item, cs); } } @@ -110,7 +140,7 @@ bool Checker::process_queue() return true; } - dispatch_item(item, cs->second); + dispatch_item(item, cs->second.second); return true; } @@ -194,7 +224,6 @@ void Checker::dispatch_single_item(const QueueItem & item, const CheckedStructur dispatch_stl_ptr_vector(item, cs); break; case IDTYPE_OPAQUE: - UNEXPECTED; break; case IDTYPE_UNION: dispatch_untagged_union(item, cs); @@ -204,27 +233,78 @@ void Checker::dispatch_single_item(const QueueItem & item, const CheckedStructur void Checker::dispatch_primitive(const QueueItem & item, const CheckedStructure & cs) { - if (cs.identity->isConstructed()) + if (cs.identity == df::identity_traits::get()) { - if (cs.identity == df::identity_traits::get()) - { - // TODO check std::string - UNEXPECTED; - } - else + check_stl_string(item); + } + else if (cs.identity == df::identity_traits::get()) + { + // TODO check c strings + UNEXPECTED; + } + else if (cs.identity == df::identity_traits::get()) + { + auto val = *reinterpret_cast(item.ptr); + if (val > 1 && val != 0xd2) { - UNEXPECTED; + FAIL("invalid value for bool: " << int(val)); } } - - // TODO: check primitives - UNEXPECTED; + else if (auto int_id = dynamic_cast(cs.identity)) + { + // TODO check ints? + } + else if (auto float_id = dynamic_cast(cs.identity)) + { + // TODO check floats? + } + else + { + UNEXPECTED; + } } void Checker::dispatch_pointer(const QueueItem & item, const CheckedStructure & cs) { - auto identity = static_cast(cs.identity); - // TODO: check pointer - UNEXPECTED; + auto target_ptr = validate_and_dereference(item); + if (!target_ptr) + { + return; + } + +#ifdef DFHACK64 + if (uintptr_t(target_ptr) == 0xd2d2d2d2d2d2d2d2) +#else + if (uintptr_t(target_ptr) == 0xd2d2d2d2) +#endif + { + return; + } + + QueueItem target_item(item.path, target_ptr); + auto target = static_cast(cs.identity)->getTarget(); + if (!target) + { + check_unknown_pointer(item); + return; + } + + // 256 is an arbitrarily chosen size threshold + if (cs.count || target->byte_size() <= 256) + { + // target is small, or we are inside an array of pointers; handle now + if (queue_item(target_item, CheckedStructure(target))) + { + // we insert it into the queue to make sure we're not stuck in a loop + // get it back out of the queue to prevent the queue growing too big + queue.pop_back(); + dispatch_item(target_item, CheckedStructure(target)); + } + } + else + { + // target is large and not part of an array; handle later + queue_item(target_item, CheckedStructure(target)); + } } void Checker::dispatch_container(const QueueItem & item, const CheckedStructure & cs) { @@ -232,11 +312,7 @@ void Checker::dispatch_container(const QueueItem & item, const CheckedStructure auto base_container = identity->getFullName(nullptr); if (base_container == "vector") { - if (identity->getIndexEnumType()) - { - UNEXPECTED; - } - check_stl_vector(item, identity->getItemType()); + check_stl_vector(item, identity->getItemType(), identity->getIndexEnumType()); } else if (base_container == "deque") { @@ -262,12 +338,10 @@ void Checker::dispatch_bit_container(const QueueItem & item, const CheckedStruct if (base_container == "BitArray<>") { // TODO: check DF bit array - UNEXPECTED; } else if (base_container == "vector") { // TODO: check stl bit vector - UNEXPECTED; } else { @@ -281,8 +355,66 @@ void Checker::dispatch_bitfield(const QueueItem & item, const CheckedStructure & return; } - // TODO: check bitfields - UNEXPECTED; + auto bitfield_type = static_cast(cs.identity); + uint64_t bitfield_value; + switch (bitfield_type->byte_size()) + { + case 1: + bitfield_value = validate_and_dereference(item); + // don't check for uninitialized; too small to be sure + break; + case 2: + bitfield_value = validate_and_dereference(item); + if (bitfield_value == 0xd2d2) + { + bitfield_value = 0; + } + break; + case 4: + bitfield_value = validate_and_dereference(item); + if (bitfield_value == 0xd2d2d2d2) + { + bitfield_value = 0; + } + break; + case 8: + bitfield_value = validate_and_dereference(item); + if (bitfield_value == 0xd2d2d2d2d2d2d2d2) + { + bitfield_value = 0; + } + break; + default: + UNEXPECTED; + bitfield_value = 0; + break; + } + + auto num_bits = bitfield_type->getNumBits(); + auto bits = bitfield_type->getBits(); + for (int i = 0; i < 64; i++) + { + if (!(num_bits & 1)) + { + bitfield_value >>= 1; + continue; + } + + bitfield_value >>= 1; + + if (i >= num_bits || !bits[i].size) + { + FAIL("bitfield bit " << i << " is out of range"); + } + else if (unnamed && bits[i].size > 0 && !bits[i].name) + { + FAIL("bitfield bit " << i << " is unnamed"); + } + else if (unnamed && !bits[i + bits[i].size].name) + { + FAIL("bitfield bit " << i << " (part of a field starting at bit " << (i + bits[i].size) << ") is unnamed"); + } + } } void Checker::dispatch_enum(const QueueItem & item, const CheckedStructure & cs) { @@ -291,8 +423,31 @@ void Checker::dispatch_enum(const QueueItem & item, const CheckedStructure & cs) return; } - // TODO: check enums - UNEXPECTED; + auto enum_type = static_cast(cs.identity); + auto enum_value = get_int_value(item, enum_type->getBaseType()); + if (enum_type->byte_size() == 2 && uint16_t(enum_value) == 0xd2d2) + { + return; + } + else if (enum_type->byte_size() == 4 && uint32_t(enum_value) == 0xd2d2d2d2) + { + return; + } + else if (enum_type->byte_size() == 8 && uint64_t(enum_value) == 0xd2d2d2d2d2d2d2d2) + { + return; + } + + auto enum_name = get_enum_item_key(enum_type, enum_value); + if (!enum_name) + { + FAIL("enum value (" << enum_value << ") is out of range"); + return; + } + if (unnamed && !*enum_name) + { + FAIL("enum value (" << enum_value << ") is unnamed"); + } } void Checker::dispatch_struct(const QueueItem & item, const CheckedStructure & cs) { @@ -300,6 +455,11 @@ void Checker::dispatch_struct(const QueueItem & item, const CheckedStructure & c for (auto p = identity; p; p = p->getParent()) { auto fields = p->getFields(); + if (!fields) + { + continue; + } + for (auto field = fields; field->mode != struct_field_info::END; field++) { dispatch_field(item, cs, fields, field); @@ -314,16 +474,28 @@ void Checker::dispatch_field(const QueueItem & item, const CheckedStructure & cs return; } + auto field_ptr = PTR_ADD(item.ptr, field->offset); + QueueItem field_item(item, field->name, field_ptr); + CheckedStructure field_cs(field); + auto tag_field = find_union_tag(fields, field); if (tag_field) { - UNEXPECTED; + auto tag_ptr = PTR_ADD(item.ptr, tag_field->offset); + QueueItem tag_item(item, tag_field->name, tag_ptr); + CheckedStructure tag_cs(tag_field); + if (tag_cs.identity->isContainer()) + { + dispatch_tagged_union_vector(field_item, tag_item, field_cs, tag_cs); + } + else + { + dispatch_tagged_union(field_item, tag_item, field_cs, tag_cs); + } return; } - auto field_ptr = PTR_ADD(item.ptr, field->offset); - CheckedStructure field_cs(field); - dispatch_item(QueueItem(item, field->name, field_ptr), field_cs); + dispatch_item(field_item, field_cs); } void Checker::dispatch_class(const QueueItem & item, const CheckedStructure & cs) { @@ -336,40 +508,158 @@ void Checker::dispatch_class(const QueueItem & item, const CheckedStructure & cs auto base_identity = static_cast(cs.identity); auto vptr = static_cast(const_cast(item.ptr)); - if (!base_identity->is_instance(vptr)) + auto identity = virtual_identity::get(vptr); + if (!identity) { - FAIL("expected subclass of " << base_identity->getFullName() << ", but got " << vtable_name); + FAIL("unidentified subclass of " << base_identity->getFullName() << ": " << vtable_name); + return; + } + if (base_identity != identity && !base_identity->is_subclass(identity)) + { + FAIL("expected subclass of " << base_identity->getFullName() << ", but got " << identity->getFullName()); return; } - auto identity = virtual_identity::get(vptr); + if (data.count(item.ptr) && data.at(item.ptr).first == item.path) + { + // TODO: handle cases where this may overlap later data + data.at(item.ptr).second.identity = identity; + } + dispatch_struct(QueueItem(item.path + "<" + identity->getFullName() + ">", item.ptr), CheckedStructure(identity)); } void Checker::dispatch_buffer(const QueueItem & item, const CheckedStructure & cs) { auto identity = static_cast(cs.identity); - if (identity->getIndexEnumType()) - { - UNEXPECTED; - } auto item_identity = identity->getItemType(); - dispatch_item(item, CheckedStructure(item_identity, identity->byte_size() / item_identity->byte_size())); + dispatch_item(item, CheckedStructure(item_identity, identity->byte_size() / item_identity->byte_size(), static_cast(identity->getIndexEnumType()))); } void Checker::dispatch_stl_ptr_vector(const QueueItem & item, const CheckedStructure & cs) { auto identity = static_cast(cs.identity); - if (identity->getIndexEnumType()) + auto ptr_type = wrap_in_pointer(identity->getItemType()); + check_stl_vector(item, ptr_type, identity->getIndexEnumType()); +} +void Checker::dispatch_tagged_union(const QueueItem & item, const QueueItem & tag_item, const CheckedStructure & cs, const CheckedStructure & tag_cs) +{ + if (tag_cs.identity->type() != IDTYPE_ENUM || cs.identity->type() != IDTYPE_UNION) + { + UNEXPECTED; + return; + } + + auto tag_identity = static_cast(tag_cs.identity); + auto tag_value = get_int_value(tag_item, tag_identity->getBaseType()); + auto tag_name = get_enum_item_key(tag_identity, tag_value); + if (!tag_name) + { + FAIL("tagged union tag (accessed as " << tag_item.path << ") value (" << tag_value << ") not defined in enum " << tag_cs.identity->getFullName()); + return; + } + + if (!*tag_name) + { + FAIL("tagged union tag (accessed as " << tag_item.path << ") value (" << tag_value << ") is unnamed"); + return; + } + + auto union_type = static_cast(cs.identity); + for (auto field = union_type->getFields(); field->mode != struct_field_info::END; field++) + { + if (strcmp(field->name, *tag_name)) + { + continue; + } + + if (field->offset != 0) + { + UNEXPECTED; + } + + dispatch_item(QueueItem(item, field->name, item.ptr), field); + return; + } + + auto union_data_ptr = reinterpret_cast(item.ptr); + uint8_t padding_byte = *union_data_ptr; + if (padding_byte == 0x00 || padding_byte == 0xd2 || padding_byte == 0xff) + { + bool all_padding = true; + for (size_t i = 0; i < union_type->byte_size(); i++) + { + if (union_data_ptr[i] != padding_byte) + { + all_padding = false; + break; + } + } + + // don't ask for fields if it's all padding + if (all_padding) + { + return; + } + } + + FAIL("tagged union missing " << *tag_name << " field to match tag (accessed as " << tag_item.path << ") value (" << tag_value << ")"); +} +void Checker::dispatch_tagged_union_vector(const QueueItem & item, const QueueItem & tag_item, const CheckedStructure & cs, const CheckedStructure & tag_cs) +{ + auto union_container_identity = static_cast(cs.identity); + CheckedStructure union_item_cs(union_container_identity->getItemType()); + if (union_container_identity->type() != IDTYPE_CONTAINER) + { + // assume pointer container + union_item_cs.identity = wrap_in_pointer(union_item_cs.identity); + } + + auto tag_container_identity = static_cast(tag_cs.identity); + auto tag_container_base = tag_container_identity->getFullName(nullptr); + if (tag_container_base == "vector") + { + auto vec_union = validate_vector_size(item, union_item_cs); + CheckedStructure tag_item_cs(tag_container_identity->getItemType()); + auto vec_tag = validate_vector_size(tag_item, tag_item_cs); + if (!vec_union.first || !vec_tag.first) + { + // invalid vectors (already warned) + return; + } + if (!vec_union.second && !vec_tag.second) + { + // empty vectors + return; + } + if (vec_union.second != vec_tag.second) + { + FAIL("tagged union vector is " << vec_union.second << " elements, but tag vector (accessed as " << tag_item.path << ") is " << vec_tag.second << " elements"); + } + + for (size_t i = 0; i < vec_union.second && i < vec_tag.second; i++) + { + dispatch_tagged_union(QueueItem(item, i, vec_union.first), QueueItem(tag_item, i, vec_tag.first), union_item_cs, tag_item_cs); + vec_union.first = PTR_ADD(vec_union.first, union_item_cs.identity->byte_size()); + vec_tag.first = PTR_ADD(vec_tag.first, tag_item_cs.identity->byte_size()); + } + } + else if (tag_container_base == "vector") + { + // TODO + UNEXPECTED; + } + else { UNEXPECTED; } - auto ptr_type = wrap_in_pointer(identity->getItemType()); - check_stl_vector(item, ptr_type); } void Checker::dispatch_untagged_union(const QueueItem & item, const CheckedStructure & cs) { + // special case for large_integer weirdness if (cs.identity == df::identity_traits::get()) { + // it's 16 bytes on 64-bit linux due to a messy header in libgraphics + // but only the first 8 bytes are ever used dispatch_primitive(item, CheckedStructure(df::identity_traits::get())); return; } @@ -377,13 +667,141 @@ void Checker::dispatch_untagged_union(const QueueItem & item, const CheckedStruc UNEXPECTED; } -void Checker::check_stl_vector(const QueueItem & item, type_identity *item_identity) +void Checker::check_unknown_pointer(const QueueItem & item) +{ + CheckedStructure cs(df::identity_traits::get()); + if (auto allocated_size = get_allocated_size(item)) + { + FAIL("pointer to a block of " << allocated_size << " bytes of allocated memory"); + if (allocated_size >= MIN_SIZE_FOR_SUGGEST && known_types_by_size.count(allocated_size)) + { + FAIL("known types of this size: " << join_strings(", ", known_types_by_size.at(allocated_size))); + } + + // check recursively if it's the right size for a pointer + // or if it starts with what might be a valid pointer + if (allocated_size == sizeof(void *) || (allocated_size > sizeof(void *) && is_valid_dereference(item, cs, true))) + { + QueueItem ptr_item(item, "?ptr?", item.ptr); + if (queue_item(ptr_item, cs)) + { + queue.pop_back(); + dispatch_pointer(ptr_item, cs); + } + } + } +#ifndef WIN32 + else if (auto str = validate_stl_string_pointer(&item.ptr)) + { + FAIL("untyped pointer is actually stl-string with value \"" << *str << "\" (length " << str->length() << ")"); + } +#endif + else if (auto vtable_name = get_vtable_name(QueueItem(item.path, &item.ptr), cs, true)) + { + FAIL("pointer to a vtable: " << vtable_name); + } + else if (sizes) + { + //FAIL("pointer to memory with no size information"); + } +} + +void Checker::check_stl_vector(const QueueItem & item, type_identity *item_identity, type_identity *eid) { auto vec_items = validate_vector_size(item, CheckedStructure(item_identity)); + + // skip bad pointer vectors + if (item.path.length() > 4 && item.path.substr(item.path.length() - 4) == ".bad" && item_identity->type() == IDTYPE_POINTER) + { + return; + } + if (vec_items.first && vec_items.second) { QueueItem items_item(item.path, vec_items.first); - CheckedStructure items_cs(item_identity, vec_items.second); + CheckedStructure items_cs(item_identity, vec_items.second, static_cast(eid)); queue_item(items_item, items_cs); } } + +void Checker::check_stl_string(const QueueItem & item) +{ + const static CheckedStructure cs(df::identity_traits::get()); + +#ifdef WIN32 + struct string_data + { + union + { + uintptr_t start; + char local_data[16]; + }; + size_t length; + size_t capacity; + }; +#else + struct string_data + { + struct string_data_inner + { + size_t length; + size_t capacity; + int32_t refcount; + } *ptr; + }; +#endif + + auto string = reinterpret_cast(item.ptr); +#ifdef WIN32 + bool is_local = string->capacity < 16; + const char *start = is_local ? &string->local_data[0] : reinterpret_cast(string->start); + ptrdiff_t length = string->length; + ptrdiff_t capacity = string->capacity; +#else + if (!is_valid_dereference(QueueItem(item, "?ptr?", string->ptr), 1)) + { + // nullptr is NOT okay here + FAIL("invalid string pointer " << stl_sprintf("%p", string->ptr)); + return; + } + if (!is_valid_dereference(QueueItem(item, "?hdr?", string->ptr - 1), sizeof(*string->ptr))) + { + return; + } + const char *start = reinterpret_cast(string->ptr); + ptrdiff_t length = (string->ptr - 1)->length; + ptrdiff_t capacity = (string->ptr - 1)->capacity; +#endif + + if (length < 0) + { + FAIL("string length is negative (" << length << ")"); + } + else if (capacity < 0) + { + FAIL("string capacity is negative (" << capacity << ")"); + } + else if (capacity < length) + { + FAIL("string capacity (" << capacity << ") is less than length (" << length << ")"); + } + +#ifndef WIN32 + const std::string empty_string; + auto empty_string_data = reinterpret_cast(&empty_string); + if (sizes && string->ptr != empty_string_data->ptr) + { + size_t allocated_size = get_allocated_size(QueueItem(item, "?hdr?", string->ptr - 1)); + size_t expected_size = sizeof(*string->ptr) + capacity + 1; + + if (!allocated_size) + { + FAIL("pointer does not appear to be a string"); + } + else if (allocated_size != expected_size) + { + FAIL("allocated string data size (" << allocated_size << ") does not match expected size (" << expected_size << ")"); + } + } +#endif +} diff --git a/plugins/devel/check-structures-sanity/main.cpp b/plugins/devel/check-structures-sanity/main.cpp index 055cbe765..8303b0b4e 100644 --- a/plugins/devel/check-structures-sanity/main.cpp +++ b/plugins/devel/check-structures-sanity/main.cpp @@ -26,6 +26,9 @@ DFhackCExport command_result plugin_init(color_ostream &, std::vectormode == struct_field_info::STATIC_ARRAY && field->eid) - { - UNEXPECTED; - } - else if (field->type && field->type->isContainer()) - { - auto expected_eid = static_cast(field->type)->getIndexEnumType(); - if (field->eid != expected_eid) - { - UNEXPECTED; - } - } - else if (field->eid) - { - UNEXPECTED; - } - identity = field->type; + eid = field->eid; switch (field->mode) { case struct_field_info::END: @@ -129,3 +120,15 @@ type_identity *Checker::wrap_in_pointer(type_identity *base) { RETURN_CACHED_WRAPPER(df::pointer_identity, base); } + +std::map> known_types_by_size; +void build_size_table() +{ + for (auto & ident : compound_identity::getTopScope()) + { + if (ident->byte_size() >= MIN_SIZE_FOR_SUGGEST) + { + known_types_by_size[ident->byte_size()].push_back(ident->getFullName()); + } + } +} diff --git a/plugins/devel/check-structures-sanity/validate.cpp b/plugins/devel/check-structures-sanity/validate.cpp index 6bae2e1bc..9b089cb79 100644 --- a/plugins/devel/check-structures-sanity/validate.cpp +++ b/plugins/devel/check-structures-sanity/validate.cpp @@ -1,9 +1,8 @@ #include "check-structures-sanity.h" -bool Checker::is_valid_dereference(const QueueItem & item, const CheckedStructure & cs, bool quiet) +bool Checker::is_valid_dereference(const QueueItem & item, const CheckedStructure & cs, size_t size, bool quiet) { auto base = const_cast(item.ptr); - auto size = cs.full_size(); if (!base) { // cannot dereference null pointer, but not an error @@ -83,6 +82,47 @@ bool Checker::is_valid_dereference(const QueueItem & item, const CheckedStructur #undef FAIL_PTR } +int64_t Checker::get_int_value(const QueueItem & item, type_identity *type, bool quiet) +{ + if (type == df::identity_traits::get()) + { + return validate_and_dereference(item, quiet); + } + else if (type == df::identity_traits::get()) + { + return validate_and_dereference(item, quiet); + } + else if (type == df::identity_traits::get()) + { + return validate_and_dereference(item, quiet); + } + else if (type == df::identity_traits::get()) + { + return validate_and_dereference(item, quiet); + } + else if (type == df::identity_traits::get()) + { + return validate_and_dereference(item, quiet); + } + else if (type == df::identity_traits::get()) + { + return int64_t(validate_and_dereference(item, quiet)); + } + else if (type == df::identity_traits::get()) + { + return validate_and_dereference(item, quiet); + } + else if (type == df::identity_traits::get()) + { + return validate_and_dereference(item, quiet); + } + else + { + UNEXPECTED; + return 0; + } +} + const char *Checker::get_vtable_name(const QueueItem & item, const CheckedStructure & cs, bool quiet) { auto vtable = validate_and_dereference(QueueItem(item, "?vtable?", item.ptr), quiet); @@ -235,3 +275,98 @@ std::pair Checker::validate_vector_size(const QueueItem & return local_ok ? std::make_pair(start_ptr, ulength / item_size) : std::make_pair(nullptr, 0); } + +size_t Checker::get_allocated_size(const QueueItem & item) +{ + if (!sizes) + { + return 0; + } + + uint32_t tag = *reinterpret_cast(PTR_ADD(item.ptr, -8)); + if (tag == 0xdfdf4ac8) + { + return *reinterpret_cast(PTR_ADD(item.ptr, -16)); + } + + return 0; +} + +#ifndef WIN32 +const std::string *Checker::validate_stl_string_pointer(const void *const* base) +{ + std::string empty_string; + if (*base == *reinterpret_cast(&empty_string)) + { + return reinterpret_cast(base); + } + + const struct string_data_inner + { + size_t length; + size_t capacity; + int32_t refcount; + } *str_data = static_cast(*base) - 1; + + uint32_t tag = *reinterpret_cast(PTR_ADD(str_data, -8)); + if (tag == 0xdfdf4ac8) + { + size_t allocated_size = *reinterpret_cast(PTR_ADD(str_data, -16)); + size_t expected_size = sizeof(*str_data) + str_data->capacity + 1; + + if (allocated_size != expected_size) + { + return nullptr; + } + } + else + { + return nullptr; + } + + if (str_data->capacity < str_data->length) + { + return nullptr; + } + + const char *ptr = reinterpret_cast(*base); + for (size_t i = 0; i < str_data->length; i++) + { + if (!*ptr++) + { + return nullptr; + } + } + + if (*ptr++) + { + return nullptr; + } + + return reinterpret_cast(base); +} +#endif + +const char *const *Checker::get_enum_item_key(enum_identity *identity, int64_t value) +{ + size_t index; + if (auto cplx = identity->getComplex()) + { + auto it = cplx->value_index_map.find(value); + if (it == cplx->value_index_map.cend()) + { + return nullptr; + } + index = it->second; + } + else + { + if (value < identity->getFirstItem() || value > identity->getLastItem()) + { + return nullptr; + } + index = value - identity->getFirstItem(); + } + + return &identity->getKeys()[index]; +} From eabff06eef0b42a91b89b2a5fb864fba96a7bd61 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Wed, 11 Mar 2020 11:20:10 -0500 Subject: [PATCH 21/32] check-structures-sanity: warn when an integer might be a pointer --- .../check-structures-sanity.h | 3 + .../check-structures-sanity/dispatch.cpp | 62 ++++++++++++++++--- .../devel/check-structures-sanity/main.cpp | 4 +- .../devel/check-structures-sanity/types.cpp | 12 ++-- .../check-structures-sanity/validate.cpp | 10 +++ 5 files changed, 76 insertions(+), 15 deletions(-) diff --git a/plugins/devel/check-structures-sanity/check-structures-sanity.h b/plugins/devel/check-structures-sanity/check-structures-sanity.h index 6946faa3f..69307098b 100644 --- a/plugins/devel/check-structures-sanity/check-structures-sanity.h +++ b/plugins/devel/check-structures-sanity/check-structures-sanity.h @@ -34,6 +34,7 @@ struct CheckedStructure type_identity *identity; size_t count; enum_identity *eid; + bool ptr_is_array; CheckedStructure(); explicit CheckedStructure(type_identity *, size_t = 0); @@ -76,6 +77,7 @@ public: bool sizes; bool unnamed; bool failfast; + bool noprogress; Checker(color_ostream & out); bool queue_item(const QueueItem & item, const CheckedStructure & cs); @@ -133,6 +135,7 @@ private: void check_unknown_pointer(const QueueItem &); void check_stl_vector(const QueueItem &, type_identity *, type_identity *); void check_stl_string(const QueueItem &); + void check_possible_pointer(const QueueItem &, const CheckedStructure &); friend struct CheckedStructure; static type_identity *wrap_in_pointer(type_identity *); diff --git a/plugins/devel/check-structures-sanity/dispatch.cpp b/plugins/devel/check-structures-sanity/dispatch.cpp index 1e02aaf9f..10fdb3ee3 100644 --- a/plugins/devel/check-structures-sanity/dispatch.cpp +++ b/plugins/devel/check-structures-sanity/dispatch.cpp @@ -11,7 +11,8 @@ Checker::Checker(color_ostream & out) : enums(false), sizes(false), unnamed(false), - failfast(false) + failfast(false), + noprogress(!out.is_console()) { Core::getInstance().p->getMemRanges(mapped); } @@ -252,6 +253,8 @@ void Checker::dispatch_primitive(const QueueItem & item, const CheckedStructure } else if (auto int_id = dynamic_cast(cs.identity)) { + check_possible_pointer(item, cs); + // TODO check ints? } else if (auto float_id = dynamic_cast(cs.identity)) @@ -288,22 +291,46 @@ void Checker::dispatch_pointer(const QueueItem & item, const CheckedStructure & return; } + CheckedStructure target_cs(target); + + if (auto allocated_size = get_allocated_size(target_item)) + { + auto expected_size = target->byte_size(); + if (target->type() == IDTYPE_CLASS && get_vtable_name(target_item, target_cs, true)) + { + auto target_virtual = virtual_identity::get(static_cast(const_cast(target_item.ptr))); + expected_size = target_virtual->byte_size(); + } + if (cs.ptr_is_array && allocated_size % expected_size == 0) + { + target_cs.count = allocated_size / expected_size; + } + else if (allocated_size > expected_size) + { + FAIL("identified structure is too small (expected " << expected_size << " bytes, but there are " << allocated_size << " bytes allocated)"); + } + else if (allocated_size < expected_size) + { + FAIL("identified structure is too big (expected " << expected_size << " bytes, but there are " << allocated_size << " bytes allocated)"); + } + } + // 256 is an arbitrarily chosen size threshold if (cs.count || target->byte_size() <= 256) { // target is small, or we are inside an array of pointers; handle now - if (queue_item(target_item, CheckedStructure(target))) + if (queue_item(target_item, target_cs)) { // we insert it into the queue to make sure we're not stuck in a loop // get it back out of the queue to prevent the queue growing too big queue.pop_back(); - dispatch_item(target_item, CheckedStructure(target)); + dispatch_item(target_item, target_cs); } } else { // target is large and not part of an array; handle later - queue_item(target_item, CheckedStructure(target)); + queue_item(target_item, target_cs); } } void Checker::dispatch_container(const QueueItem & item, const CheckedStructure & cs) @@ -350,6 +377,8 @@ void Checker::dispatch_bit_container(const QueueItem & item, const CheckedStruct } void Checker::dispatch_bitfield(const QueueItem & item, const CheckedStructure & cs) { + check_possible_pointer(item, cs); + if (!enums) { return; @@ -418,6 +447,8 @@ void Checker::dispatch_bitfield(const QueueItem & item, const CheckedStructure & } void Checker::dispatch_enum(const QueueItem & item, const CheckedStructure & cs) { + check_possible_pointer(item, cs); + if (!enums) { return; @@ -669,7 +700,7 @@ void Checker::dispatch_untagged_union(const QueueItem & item, const CheckedStruc void Checker::check_unknown_pointer(const QueueItem & item) { - CheckedStructure cs(df::identity_traits::get()); + const static CheckedStructure cs(nullptr, 0); if (auto allocated_size = get_allocated_size(item)) { FAIL("pointer to a block of " << allocated_size << " bytes of allocated memory"); @@ -680,13 +711,14 @@ void Checker::check_unknown_pointer(const QueueItem & item) // check recursively if it's the right size for a pointer // or if it starts with what might be a valid pointer - if (allocated_size == sizeof(void *) || (allocated_size > sizeof(void *) && is_valid_dereference(item, cs, true))) + QueueItem ptr_item(item, "?ptr?", item.ptr); + if (allocated_size == sizeof(void *) || (allocated_size > sizeof(void *) && is_valid_dereference(ptr_item, 1, true))) { - QueueItem ptr_item(item, "?ptr?", item.ptr); - if (queue_item(ptr_item, cs)) + CheckedStructure ptr_cs(df::identity_traits::get()); + if (queue_item(ptr_item, ptr_cs)) { queue.pop_back(); - dispatch_pointer(ptr_item, cs); + dispatch_pointer(ptr_item, ptr_cs); } } } @@ -805,3 +837,15 @@ void Checker::check_stl_string(const QueueItem & item) } #endif } +void Checker::check_possible_pointer(const QueueItem & item, const CheckedStructure & cs) +{ + if (sizes && uintptr_t(item.ptr) % sizeof(void *) == 0) + { + auto ptr = validate_and_dereference(item, true); + QueueItem ptr_item(item, "?maybe_pointer?", ptr); + if (ptr && is_valid_dereference(ptr_item, 1, true)) + { + check_unknown_pointer(ptr_item); + } + } +} diff --git a/plugins/devel/check-structures-sanity/main.cpp b/plugins/devel/check-structures-sanity/main.cpp index 8303b0b4e..f29e8f201 100644 --- a/plugins/devel/check-structures-sanity/main.cpp +++ b/plugins/devel/check-structures-sanity/main.cpp @@ -37,6 +37,7 @@ static command_result command(color_ostream & out, std::vector & pa CoreSuspender suspend; Checker checker(out); + // check parameters with values first #define VAL_PARAM(name, expr_using_value) \ auto name ## _idx = std::find(parameters.begin(), parameters.end(), "-" #name); \ @@ -73,6 +74,7 @@ static command_result command(color_ostream & out, std::vector & pa BOOL_PARAM(sizes); BOOL_PARAM(unnamed); BOOL_PARAM(failfast); + BOOL_PARAM(noprogress); #undef BOOL_PARAM if (parameters.size() > 1) @@ -117,7 +119,7 @@ static command_result command(color_ostream & out, std::vector & pa while (checker.process_queue()) { - if (out.is_console()) + if (!checker.noprogress) { out << "checked " << checker.checked_count << " fields\r" << std::flush; } diff --git a/plugins/devel/check-structures-sanity/types.cpp b/plugins/devel/check-structures-sanity/types.cpp index dc3ccb390..90e361be8 100644 --- a/plugins/devel/check-structures-sanity/types.cpp +++ b/plugins/devel/check-structures-sanity/types.cpp @@ -19,15 +19,14 @@ CheckedStructure::CheckedStructure() : { } CheckedStructure::CheckedStructure(type_identity *identity, size_t count) : - identity(identity), - count(count), - eid(nullptr) + CheckedStructure(identity, count, nullptr) { } CheckedStructure::CheckedStructure(type_identity *identity, size_t count, enum_identity *eid) : identity(identity), count(count), - eid(eid) + eid(eid), + ptr_is_array(false) { } CheckedStructure::CheckedStructure(const struct_field_info *field) : @@ -60,7 +59,10 @@ CheckedStructure::CheckedStructure(const struct_field_info *field) : count = field->count; break; case struct_field_info::POINTER: - // TODO: check flags (stored in field->count) + if (field->count & PTRFLAG_IS_ARRAY) + { + ptr_is_array = true; + } identity = Checker::wrap_in_pointer(field->type); break; case struct_field_info::STATIC_ARRAY: diff --git a/plugins/devel/check-structures-sanity/validate.cpp b/plugins/devel/check-structures-sanity/validate.cpp index 9b089cb79..8249a169d 100644 --- a/plugins/devel/check-structures-sanity/validate.cpp +++ b/plugins/devel/check-structures-sanity/validate.cpp @@ -283,6 +283,11 @@ size_t Checker::get_allocated_size(const QueueItem & item) return 0; } + if (uintptr_t(item.ptr) % 32 != 16) + { + return 0; + } + uint32_t tag = *reinterpret_cast(PTR_ADD(item.ptr, -8)); if (tag == 0xdfdf4ac8) { @@ -308,6 +313,11 @@ const std::string *Checker::validate_stl_string_pointer(const void *const* base) int32_t refcount; } *str_data = static_cast(*base) - 1; + if (!is_valid_dereference(QueueItem("str", PTR_ADD(str_data, -16)), 16, true)) + { + return nullptr; + } + uint32_t tag = *reinterpret_cast(PTR_ADD(str_data, -8)); if (tag == 0xdfdf4ac8) { From 469c49c8b9ec269eaf1fdf3265002b10e96918d8 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Wed, 11 Mar 2020 17:49:34 -0500 Subject: [PATCH 22/32] check-structures-sanity: keep track of whether data is within a larger structure --- .../check-structures-sanity.h | 6 +- .../check-structures-sanity/dispatch.cpp | 81 +++++++++++-------- .../devel/check-structures-sanity/types.cpp | 9 ++- .../check-structures-sanity/validate.cpp | 16 +++- 4 files changed, 70 insertions(+), 42 deletions(-) diff --git a/plugins/devel/check-structures-sanity/check-structures-sanity.h b/plugins/devel/check-structures-sanity/check-structures-sanity.h index 69307098b..b173c45a0 100644 --- a/plugins/devel/check-structures-sanity/check-structures-sanity.h +++ b/plugins/devel/check-structures-sanity/check-structures-sanity.h @@ -33,12 +33,14 @@ struct CheckedStructure { type_identity *identity; size_t count; + size_t allocated_count; enum_identity *eid; bool ptr_is_array; + bool inside_structure; CheckedStructure(); explicit CheckedStructure(type_identity *, size_t = 0); - CheckedStructure(type_identity *, size_t, enum_identity *); + CheckedStructure(type_identity *, size_t, enum_identity *, bool); CheckedStructure(const struct_field_info *); size_t full_size() const; @@ -105,7 +107,7 @@ public: } int64_t get_int_value(const QueueItem & item, type_identity *type, bool quiet = false); const char *get_vtable_name(const QueueItem & item, const CheckedStructure & cs, bool quiet = false); - std::pair validate_vector_size(const QueueItem & item, const CheckedStructure & cs, bool quiet = false); + std::pair validate_vector_size(const QueueItem & item, const CheckedStructure & cs, bool quiet = false); size_t get_allocated_size(const QueueItem & item); #ifndef WIN32 // this function doesn't make sense on windows, where std::string is not pointer-sized. diff --git a/plugins/devel/check-structures-sanity/dispatch.cpp b/plugins/devel/check-structures-sanity/dispatch.cpp index 10fdb3ee3..7c290e853 100644 --- a/plugins/devel/check-structures-sanity/dispatch.cpp +++ b/plugins/devel/check-structures-sanity/dispatch.cpp @@ -160,6 +160,44 @@ void Checker::dispatch_item(const QueueItem & base, const CheckedStructure & cs) return; } + if (sizes && !cs.inside_structure) + { + if (auto allocated_size = get_allocated_size(base)) + { + auto expected_size = cs.identity->byte_size(); + if (cs.allocated_count) + expected_size *= cs.allocated_count; + else if (cs.count) + expected_size *= cs.count; + + if (cs.identity->type() == IDTYPE_CLASS && get_vtable_name(base, cs, true)) + { + if (cs.count) + { + UNEXPECTED; + } + + auto virtual_type = virtual_identity::get(static_cast(const_cast(base.ptr))); + expected_size = virtual_type->byte_size(); + } + + auto & item = base; + + if (allocated_size > expected_size) + { + FAIL("identified structure is too small (expected " << expected_size << " bytes, but there are " << allocated_size << " bytes allocated)"); + } + else if (allocated_size < expected_size) + { + FAIL("identified structure is too big (expected " << expected_size << " bytes, but there are " << allocated_size << " bytes allocated)"); + } + } + else + { + UNEXPECTED; + } + } + auto ptr = base.ptr; auto size = cs.identity->byte_size(); for (size_t i = 0; i < cs.count; i++) @@ -293,28 +331,6 @@ void Checker::dispatch_pointer(const QueueItem & item, const CheckedStructure & CheckedStructure target_cs(target); - if (auto allocated_size = get_allocated_size(target_item)) - { - auto expected_size = target->byte_size(); - if (target->type() == IDTYPE_CLASS && get_vtable_name(target_item, target_cs, true)) - { - auto target_virtual = virtual_identity::get(static_cast(const_cast(target_item.ptr))); - expected_size = target_virtual->byte_size(); - } - if (cs.ptr_is_array && allocated_size % expected_size == 0) - { - target_cs.count = allocated_size / expected_size; - } - else if (allocated_size > expected_size) - { - FAIL("identified structure is too small (expected " << expected_size << " bytes, but there are " << allocated_size << " bytes allocated)"); - } - else if (allocated_size < expected_size) - { - FAIL("identified structure is too big (expected " << expected_size << " bytes, but there are " << allocated_size << " bytes allocated)"); - } - } - // 256 is an arbitrarily chosen size threshold if (cs.count || target->byte_size() <= 256) { @@ -564,7 +580,7 @@ void Checker::dispatch_buffer(const QueueItem & item, const CheckedStructure & c auto identity = static_cast(cs.identity); auto item_identity = identity->getItemType(); - dispatch_item(item, CheckedStructure(item_identity, identity->byte_size() / item_identity->byte_size(), static_cast(identity->getIndexEnumType()))); + dispatch_item(item, CheckedStructure(item_identity, identity->byte_size() / item_identity->byte_size(), static_cast(identity->getIndexEnumType()), cs.inside_structure)); } void Checker::dispatch_stl_ptr_vector(const QueueItem & item, const CheckedStructure & cs) { @@ -657,17 +673,17 @@ void Checker::dispatch_tagged_union_vector(const QueueItem & item, const QueueIt // invalid vectors (already warned) return; } - if (!vec_union.second && !vec_tag.second) + if (!vec_union.second.count && !vec_tag.second.count) { // empty vectors return; } - if (vec_union.second != vec_tag.second) + if (vec_union.second.count != vec_tag.second.count) { - FAIL("tagged union vector is " << vec_union.second << " elements, but tag vector (accessed as " << tag_item.path << ") is " << vec_tag.second << " elements"); + FAIL("tagged union vector is " << vec_union.second.count << " elements, but tag vector (accessed as " << tag_item.path << ") is " << vec_tag.second.count << " elements"); } - for (size_t i = 0; i < vec_union.second && i < vec_tag.second; i++) + for (size_t i = 0; i < vec_union.second.count && i < vec_tag.second.count; i++) { dispatch_tagged_union(QueueItem(item, i, vec_union.first), QueueItem(tag_item, i, vec_tag.first), union_item_cs, tag_item_cs); vec_union.first = PTR_ADD(vec_union.first, union_item_cs.identity->byte_size()); @@ -691,7 +707,7 @@ void Checker::dispatch_untagged_union(const QueueItem & item, const CheckedStruc { // it's 16 bytes on 64-bit linux due to a messy header in libgraphics // but only the first 8 bytes are ever used - dispatch_primitive(item, CheckedStructure(df::identity_traits::get())); + dispatch_primitive(item, CheckedStructure(df::identity_traits::get(), 0, nullptr, cs.inside_structure)); return; } @@ -700,7 +716,7 @@ void Checker::dispatch_untagged_union(const QueueItem & item, const CheckedStruc void Checker::check_unknown_pointer(const QueueItem & item) { - const static CheckedStructure cs(nullptr, 0); + const static CheckedStructure cs(nullptr, 0, nullptr, true); if (auto allocated_size = get_allocated_size(item)) { FAIL("pointer to a block of " << allocated_size << " bytes of allocated memory"); @@ -748,17 +764,16 @@ void Checker::check_stl_vector(const QueueItem & item, type_identity *item_ident return; } - if (vec_items.first && vec_items.second) + if (vec_items.first && vec_items.second.count) { QueueItem items_item(item.path, vec_items.first); - CheckedStructure items_cs(item_identity, vec_items.second, static_cast(eid)); - queue_item(items_item, items_cs); + queue_item(items_item, vec_items.second); } } void Checker::check_stl_string(const QueueItem & item) { - const static CheckedStructure cs(df::identity_traits::get()); + const static CheckedStructure cs(df::identity_traits::get(), 0, nullptr, true); #ifdef WIN32 struct string_data diff --git a/plugins/devel/check-structures-sanity/types.cpp b/plugins/devel/check-structures-sanity/types.cpp index 90e361be8..045bca2d0 100644 --- a/plugins/devel/check-structures-sanity/types.cpp +++ b/plugins/devel/check-structures-sanity/types.cpp @@ -19,14 +19,16 @@ CheckedStructure::CheckedStructure() : { } CheckedStructure::CheckedStructure(type_identity *identity, size_t count) : - CheckedStructure(identity, count, nullptr) + CheckedStructure(identity, count, nullptr, false) { } -CheckedStructure::CheckedStructure(type_identity *identity, size_t count, enum_identity *eid) : +CheckedStructure::CheckedStructure(type_identity *identity, size_t count, enum_identity *eid, bool inside_structure) : identity(identity), count(count), + allocated_count(0), eid(eid), - ptr_is_array(false) + ptr_is_array(false), + inside_structure(inside_structure) { } CheckedStructure::CheckedStructure(const struct_field_info *field) : @@ -39,6 +41,7 @@ CheckedStructure::CheckedStructure(const struct_field_info *field) : identity = field->type; eid = field->eid; + inside_structure = true; switch (field->mode) { case struct_field_info::END: diff --git a/plugins/devel/check-structures-sanity/validate.cpp b/plugins/devel/check-structures-sanity/validate.cpp index 8249a169d..cdf0fe64e 100644 --- a/plugins/devel/check-structures-sanity/validate.cpp +++ b/plugins/devel/check-structures-sanity/validate.cpp @@ -191,8 +191,9 @@ const char *Checker::get_vtable_name(const QueueItem & item, const CheckedStruct return nullptr; } -std::pair Checker::validate_vector_size(const QueueItem & item, const CheckedStructure & cs, bool quiet) +std::pair Checker::validate_vector_size(const QueueItem & item, const CheckedStructure & cs, bool quiet) { + using ret_type = std::pair; struct vector_data { uintptr_t start; @@ -206,7 +207,7 @@ std::pair Checker::validate_vector_size(const QueueItem & ptrdiff_t capacity = vector.end_of_storage - vector.start; bool local_ok = true; - auto item_size = cs.full_size(); + auto item_size = cs.identity ? cs.identity->byte_size() : 0; if (!item_size) { item_size = 1; @@ -263,7 +264,7 @@ std::pair Checker::validate_vector_size(const QueueItem & { FAIL("vector has null pointer but capacity " << (capacity / item_size)); } - return std::make_pair(nullptr, 0); + return ret_type(); } auto start_ptr = reinterpret_cast(vector.start); @@ -273,7 +274,14 @@ std::pair Checker::validate_vector_size(const QueueItem & local_ok = false; } - return local_ok ? std::make_pair(start_ptr, ulength / item_size) : std::make_pair(nullptr, 0); + if (!local_ok) + { + return ret_type(); + } + + CheckedStructure ret_cs(cs.identity, ulength / item_size); + ret_cs.allocated_count = ucapacity / item_size; + return std::make_pair(start_ptr, ret_cs); } size_t Checker::get_allocated_size(const QueueItem & item) From 08d2ed577e5af24a52cc981757b8ef33cd586c11 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Thu, 12 Mar 2020 10:02:21 -0500 Subject: [PATCH 23/32] fix check-structures-sanity compile on windows --- .../check-structures-sanity/dispatch.cpp | 2 +- .../devel/check-structures-sanity/types.cpp | 25 +++++++++++-------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/plugins/devel/check-structures-sanity/dispatch.cpp b/plugins/devel/check-structures-sanity/dispatch.cpp index 7c290e853..bf06deb3c 100644 --- a/plugins/devel/check-structures-sanity/dispatch.cpp +++ b/plugins/devel/check-structures-sanity/dispatch.cpp @@ -580,7 +580,7 @@ void Checker::dispatch_buffer(const QueueItem & item, const CheckedStructure & c auto identity = static_cast(cs.identity); auto item_identity = identity->getItemType(); - dispatch_item(item, CheckedStructure(item_identity, identity->byte_size() / item_identity->byte_size(), static_cast(identity->getIndexEnumType()), cs.inside_structure)); + dispatch_item(item, CheckedStructure(item_identity, identity->byte_size() / item_identity->byte_size(), static_cast(identity->getIndexEnumType()), true)); } void Checker::dispatch_stl_ptr_vector(const QueueItem & item, const CheckedStructure & cs) { diff --git a/plugins/devel/check-structures-sanity/types.cpp b/plugins/devel/check-structures-sanity/types.cpp index 045bca2d0..4c4529706 100644 --- a/plugins/devel/check-structures-sanity/types.cpp +++ b/plugins/devel/check-structures-sanity/types.cpp @@ -107,23 +107,26 @@ size_t CheckedStructure::full_size() const return size; } -#define RETURN_CACHED_WRAPPER(T, base, ...) \ - static std::map> wrappers; \ - auto it = wrappers.find(base); \ - if (it != wrappers.end()) \ - { \ - return it->second.get(); \ - } \ - return (wrappers[base] = dts::make_unique(base __VA_OPT__(,) __VA_ARGS__)).get() - type_identity *Checker::wrap_in_stl_ptr_vector(type_identity *base) { - RETURN_CACHED_WRAPPER(df::stl_ptr_vector_identity, base, nullptr); + static std::map> wrappers; + auto it = wrappers.find(base); + if (it != wrappers.end()) + { + return it->second.get(); + } + return (wrappers[base] = dts::make_unique(base, nullptr)).get(); } type_identity *Checker::wrap_in_pointer(type_identity *base) { - RETURN_CACHED_WRAPPER(df::pointer_identity, base); + static std::map> wrappers; + auto it = wrappers.find(base); + if (it != wrappers.end()) + { + return it->second.get(); + } + return (wrappers[base] = dts::make_unique(base)).get(); } std::map> known_types_by_size; From 616a57224fdb26eaa3fd56e959abb18f611ca9c2 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Thu, 12 Mar 2020 11:35:52 -0500 Subject: [PATCH 24/32] check-structures-sanity: don't report possible pointers by default --- .../devel/check-structures-sanity/check-structures-sanity.h | 1 + plugins/devel/check-structures-sanity/dispatch.cpp | 5 +++-- plugins/devel/check-structures-sanity/main.cpp | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/plugins/devel/check-structures-sanity/check-structures-sanity.h b/plugins/devel/check-structures-sanity/check-structures-sanity.h index b173c45a0..7db57a6c6 100644 --- a/plugins/devel/check-structures-sanity/check-structures-sanity.h +++ b/plugins/devel/check-structures-sanity/check-structures-sanity.h @@ -80,6 +80,7 @@ public: bool unnamed; bool failfast; bool noprogress; + bool maybepointer; Checker(color_ostream & out); bool queue_item(const QueueItem & item, const CheckedStructure & cs); diff --git a/plugins/devel/check-structures-sanity/dispatch.cpp b/plugins/devel/check-structures-sanity/dispatch.cpp index bf06deb3c..77b16e39e 100644 --- a/plugins/devel/check-structures-sanity/dispatch.cpp +++ b/plugins/devel/check-structures-sanity/dispatch.cpp @@ -12,7 +12,8 @@ Checker::Checker(color_ostream & out) : sizes(false), unnamed(false), failfast(false), - noprogress(!out.is_console()) + noprogress(!out.is_console()), + maybepointer(false) { Core::getInstance().p->getMemRanges(mapped); } @@ -854,7 +855,7 @@ void Checker::check_stl_string(const QueueItem & item) } void Checker::check_possible_pointer(const QueueItem & item, const CheckedStructure & cs) { - if (sizes && uintptr_t(item.ptr) % sizeof(void *) == 0) + if (sizes && maybepointer && uintptr_t(item.ptr) % sizeof(void *) == 0) { auto ptr = validate_and_dereference(item, true); QueueItem ptr_item(item, "?maybe_pointer?", ptr); diff --git a/plugins/devel/check-structures-sanity/main.cpp b/plugins/devel/check-structures-sanity/main.cpp index f29e8f201..c503412f6 100644 --- a/plugins/devel/check-structures-sanity/main.cpp +++ b/plugins/devel/check-structures-sanity/main.cpp @@ -21,6 +21,7 @@ DFhackCExport command_result plugin_init(color_ostream &, std::vector & pa BOOL_PARAM(unnamed); BOOL_PARAM(failfast); BOOL_PARAM(noprogress); + BOOL_PARAM(maybepointer); #undef BOOL_PARAM if (parameters.size() > 1) From 2a6b2cacbdcf71ce4ded42697a2b1f7bf93d5a7c Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Thu, 12 Mar 2020 12:34:57 -0500 Subject: [PATCH 25/32] update scripts and structures --- library/xml | 2 +- scripts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/library/xml b/library/xml index e2fafcd39..3834b0a9a 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit e2fafcd396f71a1b67f74e8f2e9b7c51d2fbadf3 +Subproject commit 3834b0a9ae00ee22dde38972db25e5adaf58088a diff --git a/scripts b/scripts index a1ae6a095..5e8b1416c 160000 --- a/scripts +++ b/scripts @@ -1 +1 @@ -Subproject commit a1ae6a095b626c2667519176a28a40d5214e6fb9 +Subproject commit 5e8b1416cdb7b715e8ff345dfaf7538a599c3c3e From f204f14e522e46b4823bdf0390afed4f1d2090d2 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Thu, 12 Mar 2020 12:53:20 -0500 Subject: [PATCH 26/32] update scripts and structures --- library/xml | 2 +- scripts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/library/xml b/library/xml index 3834b0a9a..ebee452e8 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit 3834b0a9ae00ee22dde38972db25e5adaf58088a +Subproject commit ebee452e8a2198ea212a1f418c2a56fc8716a6cd diff --git a/scripts b/scripts index 5e8b1416c..54cdfb58b 160000 --- a/scripts +++ b/scripts @@ -1 +1 @@ -Subproject commit 5e8b1416cdb7b715e8ff345dfaf7538a599c3c3e +Subproject commit 54cdfb58b9dd0606fb43117df0a8f5fa5b3a3016 From 1a4c4a5f3382b6453644ed63cfc514678996b232 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Thu, 12 Mar 2020 14:03:23 -0500 Subject: [PATCH 27/32] update structures --- library/xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/xml b/library/xml index ebee452e8..08e114638 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit ebee452e8a2198ea212a1f418c2a56fc8716a6cd +Subproject commit 08e114638f3785fd2bee01714be9eca3c1bc99a5 From d456e3db7aeb844acd094d4885c9b10ada0c4918 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Thu, 12 Mar 2020 15:20:46 -0500 Subject: [PATCH 28/32] check-structures-sanity: fix unknown pointer type not actually dereferencing the pointer basic attempt at handling overlapping structs --- .../check-structures-sanity.h | 2 + .../check-structures-sanity/dispatch.cpp | 27 ++++++++++--- .../devel/check-structures-sanity/types.cpp | 39 +++++++++++++++++++ .../check-structures-sanity/validate.cpp | 16 ++++++++ 4 files changed, 78 insertions(+), 6 deletions(-) diff --git a/plugins/devel/check-structures-sanity/check-structures-sanity.h b/plugins/devel/check-structures-sanity/check-structures-sanity.h index 7db57a6c6..8a68d39e4 100644 --- a/plugins/devel/check-structures-sanity/check-structures-sanity.h +++ b/plugins/devel/check-structures-sanity/check-structures-sanity.h @@ -44,6 +44,7 @@ struct CheckedStructure CheckedStructure(const struct_field_info *); size_t full_size() const; + const struct_field_info *find_field_at_offset_with_type(size_t, const CheckedStructure &) const; }; #define MIN_SIZE_FOR_SUGGEST 64 @@ -87,6 +88,7 @@ public: void queue_globals(); bool process_queue(); + bool is_in_global(const QueueItem & item); bool is_valid_dereference(const QueueItem & item, const CheckedStructure & cs, size_t size, bool quiet); inline bool is_valid_dereference(const QueueItem & item, const CheckedStructure & cs, bool quiet = false) { diff --git a/plugins/devel/check-structures-sanity/dispatch.cpp b/plugins/devel/check-structures-sanity/dispatch.cpp index 77b16e39e..c2b7beada 100644 --- a/plugins/devel/check-structures-sanity/dispatch.cpp +++ b/plugins/devel/check-structures-sanity/dispatch.cpp @@ -78,15 +78,21 @@ bool Checker::queue_item(const QueueItem & item, const CheckedStructure & cs) } } - auto overlap = data.lower_bound(item.ptr); + auto overlap_start = data.lower_bound(item.ptr); auto overlap_end = data.lower_bound(ptr_end); - while (overlap != overlap_end) + for (auto overlap = overlap_start; overlap != overlap_end; overlap++) { - // TODO - FAIL("TODO: handle merging structures: " << overlap->second.first << " overlaps " << item.path << " (forward)"); - overlap++; + auto offset = uintptr_t(overlap->first) - uintptr_t(item.ptr); + if (!cs.find_field_at_offset_with_type(offset, overlap->second.second)) + { + // TODO + FAIL("TODO: handle merging structures: " << overlap->second.first << " overlaps " << item.path << " (forward)"); + return false; + } } + data.erase(overlap_start, overlap_end); + data[item.ptr] = std::make_pair(item.path, cs); queue.push_back(item); return true; @@ -326,7 +332,7 @@ void Checker::dispatch_pointer(const QueueItem & item, const CheckedStructure & auto target = static_cast(cs.identity)->getTarget(); if (!target) { - check_unknown_pointer(item); + check_unknown_pointer(target_item); return; } @@ -362,6 +368,10 @@ void Checker::dispatch_container(const QueueItem & item, const CheckedStructure { // TODO: check deque? } + else if (base_container == "DfArray") + { + // TODO: check DfArray + } else { UNEXPECTED; @@ -486,6 +496,11 @@ void Checker::dispatch_enum(const QueueItem & item, const CheckedStructure & cs) return; } + if (is_in_global(item) && enum_value == 0) + { + return; + } + auto enum_name = get_enum_item_key(enum_type, enum_value); if (!enum_name) { diff --git a/plugins/devel/check-structures-sanity/types.cpp b/plugins/devel/check-structures-sanity/types.cpp index 4c4529706..4f5e8ca3a 100644 --- a/plugins/devel/check-structures-sanity/types.cpp +++ b/plugins/devel/check-structures-sanity/types.cpp @@ -107,6 +107,45 @@ size_t CheckedStructure::full_size() const return size; } +const struct_field_info *CheckedStructure::find_field_at_offset_with_type(size_t offset, const CheckedStructure & type) const +{ + if (!identity) + return nullptr; + + if (offset >= identity->byte_size() && offset < full_size()) + offset %= identity->byte_size(); + else if (offset >= identity->byte_size()) + return nullptr; + + auto st = dynamic_cast(identity); + if (!st) + { + UNEXPECTED; + return nullptr; + } + + for (auto p = st; p; p = p->getParent()) + { + auto fields = p->getFields(); + if (!fields) + continue; + + for (auto field = fields; field->mode != struct_field_info::END; field++) + { + if (field->offset > offset) + continue; + + if (field->offset == offset && CheckedStructure(field).identity == type.identity) + return field; + + if (auto subfield = CheckedStructure(field).find_field_at_offset_with_type(offset - field->offset, type)) + return subfield; + } + } + + return nullptr; +} + type_identity *Checker::wrap_in_stl_ptr_vector(type_identity *base) { static std::map> wrappers; diff --git a/plugins/devel/check-structures-sanity/validate.cpp b/plugins/devel/check-structures-sanity/validate.cpp index cdf0fe64e..a3ed03980 100644 --- a/plugins/devel/check-structures-sanity/validate.cpp +++ b/plugins/devel/check-structures-sanity/validate.cpp @@ -1,5 +1,21 @@ #include "check-structures-sanity.h" +bool Checker::is_in_global(const QueueItem & item) +{ + auto fields = df::global::_identity.getFields(); + for (auto field = fields; field->mode != struct_field_info::END; field++) + { + size_t size = CheckedStructure(field).full_size(); + auto start = *reinterpret_cast(field->offset); + auto offset = uintptr_t(item.ptr) - uintptr_t(start); + if (offset < size) + { + return true; + } + } + + return false; +} bool Checker::is_valid_dereference(const QueueItem & item, const CheckedStructure & cs, size_t size, bool quiet) { auto base = const_cast(item.ptr); From 39486083f5c5d823353be9567363c1530bbe7840 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Thu, 12 Mar 2020 22:16:30 -0500 Subject: [PATCH 29/32] handle overlapping structures in check-structures-sanity better update structures --- library/xml | 2 +- .../check-structures-sanity.h | 4 +- .../check-structures-sanity/dispatch.cpp | 134 +++++++++++------- .../devel/check-structures-sanity/types.cpp | 29 ++-- 4 files changed, 106 insertions(+), 63 deletions(-) diff --git a/library/xml b/library/xml index 08e114638..8ec203292 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit 08e114638f3785fd2bee01714be9eca3c1bc99a5 +Subproject commit 8ec203292e0aff9dce2a514d4bee01ba722f16de diff --git a/plugins/devel/check-structures-sanity/check-structures-sanity.h b/plugins/devel/check-structures-sanity/check-structures-sanity.h index 8a68d39e4..45d882fef 100644 --- a/plugins/devel/check-structures-sanity/check-structures-sanity.h +++ b/plugins/devel/check-structures-sanity/check-structures-sanity.h @@ -44,7 +44,7 @@ struct CheckedStructure CheckedStructure(const struct_field_info *); size_t full_size() const; - const struct_field_info *find_field_at_offset_with_type(size_t, const CheckedStructure &) const; + bool has_type_at_offset(const CheckedStructure &, size_t) const; }; #define MIN_SIZE_FOR_SUGGEST 64 @@ -84,7 +84,7 @@ public: bool maybepointer; Checker(color_ostream & out); - bool queue_item(const QueueItem & item, const CheckedStructure & cs); + bool queue_item(const QueueItem & item, CheckedStructure cs); void queue_globals(); bool process_queue(); diff --git a/plugins/devel/check-structures-sanity/dispatch.cpp b/plugins/devel/check-structures-sanity/dispatch.cpp index c2b7beada..1fd85ce52 100644 --- a/plugins/devel/check-structures-sanity/dispatch.cpp +++ b/plugins/devel/check-structures-sanity/dispatch.cpp @@ -30,52 +30,50 @@ color_ostream & Checker::fail(int line, const QueueItem & item, const CheckedStr return out; } -bool Checker::queue_item(const QueueItem & item, const CheckedStructure & cs) +bool Checker::queue_item(const QueueItem & item, CheckedStructure cs) { - if (data.count(item.ptr)) + if (!cs.identity) { - // already checked - auto existing = data.at(item.ptr); - if (cs.identity != existing.second.identity) + UNEXPECTED; + return false; + } + + if (cs.identity->type() == IDTYPE_CLASS) + { + if (cs.count) + { + UNEXPECTED; + } + + if (get_vtable_name(item, cs, true)) { - if (cs.identity->type() == IDTYPE_CLASS && existing.second.identity->type() == IDTYPE_CLASS && get_vtable_name(item, cs, false) && static_cast(cs.identity)->is_instance(static_cast(const_cast(item.ptr)))) + auto actual_identity = virtual_identity::get(reinterpret_cast(const_cast(item.ptr))); + if (static_cast(cs.identity)->is_subclass(actual_identity)) { - return false; + cs.identity = actual_identity; } - - FAIL("TODO: handle merging structures: " << data.at(item.ptr).first << " overlaps " << item.path << " (same pointer)"); } - return false; } auto ptr_end = PTR_ADD(item.ptr, cs.full_size()); auto prev = data.lower_bound(item.ptr); - if (prev != data.cbegin()) + if (prev != data.cbegin() && uintptr_t(prev->first) > uintptr_t(item.ptr)) { prev--; - if (uintptr_t(prev->first) + prev->second.second.full_size() > uintptr_t(item.ptr)) + } + if (prev != data.cend() && uintptr_t(prev->first) <= uintptr_t(item.ptr) && uintptr_t(prev->first) + prev->second.second.full_size() > uintptr_t(item.ptr)) + { + auto offset = uintptr_t(item.ptr) - uintptr_t(prev->first); + if (!prev->second.second.has_type_at_offset(cs, offset)) { - // placeholder algorithm - if (auto sid = dynamic_cast(prev->second.second.identity)) - { - auto offset = uintptr_t(item.ptr) - uintptr_t(prev->first); - for (auto field = sid->getFields(); field->mode != struct_field_info::END; field++) - { - if (field->offset == offset) - { - if (field->mode == struct_field_info::SUBSTRUCT && field->type == cs.identity) - { - return false; - } - - UNEXPECTED; - } - } - } // TODO - FAIL("TODO: handle merging structures: " << prev->second.first << " overlaps " << item.path << " (backward)"); + FAIL("TODO: handle merging structures: " << item.path << " overlaps " << prev->second.first << " (backward)"); + return false; } + + // we've already checked this structure, or we're currently queued to do so + return false; } auto overlap_start = data.lower_bound(item.ptr); @@ -83,7 +81,7 @@ bool Checker::queue_item(const QueueItem & item, const CheckedStructure & cs) for (auto overlap = overlap_start; overlap != overlap_end; overlap++) { auto offset = uintptr_t(overlap->first) - uintptr_t(item.ptr); - if (!cs.find_field_at_offset_with_type(offset, overlap->second.second)) + if (!cs.has_type_at_offset(overlap->second.second, offset)) { // TODO FAIL("TODO: handle merging structures: " << overlap->second.first << " overlaps " << item.path << " (forward)"); @@ -612,8 +610,60 @@ void Checker::dispatch_tagged_union(const QueueItem & item, const QueueItem & ta return; } + auto union_type = static_cast(cs.identity); + auto union_data_ptr = reinterpret_cast(item.ptr); + uint8_t padding_byte = *union_data_ptr; + bool all_padding = false; + if (padding_byte == 0x00 || padding_byte == 0xd2 || padding_byte == 0xff) + { + all_padding = true; + for (size_t i = 0; i < union_type->byte_size(); i++) + { + if (union_data_ptr[i] != padding_byte) + { + all_padding = false; + break; + } + } + } + auto tag_identity = static_cast(tag_cs.identity); auto tag_value = get_int_value(tag_item, tag_identity->getBaseType()); + if (all_padding && padding_byte == 0xd2) + { + // special case: completely uninitialized + switch (tag_identity->byte_size()) + { + case 1: + if (tag_value == 0xd2) + { + return; + } + break; + case 2: + if (tag_value == 0xd2d2) + { + return; + } + break; + case 4: + if (tag_value == 0xd2d2d2d2) + { + return; + } + break; + case 8: + if (uint64_t(tag_value) == 0xd2d2d2d2d2d2d2d2) + { + return; + } + break; + default: + UNEXPECTED; + break; + } + } + auto tag_name = get_enum_item_key(tag_identity, tag_value); if (!tag_name) { @@ -627,7 +677,6 @@ void Checker::dispatch_tagged_union(const QueueItem & item, const QueueItem & ta return; } - auto union_type = static_cast(cs.identity); for (auto field = union_type->getFields(); field->mode != struct_field_info::END; field++) { if (strcmp(field->name, *tag_name)) @@ -644,25 +693,10 @@ void Checker::dispatch_tagged_union(const QueueItem & item, const QueueItem & ta return; } - auto union_data_ptr = reinterpret_cast(item.ptr); - uint8_t padding_byte = *union_data_ptr; - if (padding_byte == 0x00 || padding_byte == 0xd2 || padding_byte == 0xff) + // don't ask for fields if it's all padding + if (all_padding) { - bool all_padding = true; - for (size_t i = 0; i < union_type->byte_size(); i++) - { - if (union_data_ptr[i] != padding_byte) - { - all_padding = false; - break; - } - } - - // don't ask for fields if it's all padding - if (all_padding) - { - return; - } + return; } FAIL("tagged union missing " << *tag_name << " field to match tag (accessed as " << tag_item.path << ") value (" << tag_value << ")"); diff --git a/plugins/devel/check-structures-sanity/types.cpp b/plugins/devel/check-structures-sanity/types.cpp index 4f5e8ca3a..24d33301e 100644 --- a/plugins/devel/check-structures-sanity/types.cpp +++ b/plugins/devel/check-structures-sanity/types.cpp @@ -107,21 +107,30 @@ size_t CheckedStructure::full_size() const return size; } -const struct_field_info *CheckedStructure::find_field_at_offset_with_type(size_t offset, const CheckedStructure & type) const +bool CheckedStructure::has_type_at_offset(const CheckedStructure & type, size_t offset) const { if (!identity) - return nullptr; + return false; + + if (offset == 0 && identity == type.identity) + return true; if (offset >= identity->byte_size() && offset < full_size()) offset %= identity->byte_size(); else if (offset >= identity->byte_size()) - return nullptr; + return false; + + if (identity->type() == IDTYPE_BUFFER) + { + auto target = static_cast(identity)->getItemType(); + return CheckedStructure(target, 0).has_type_at_offset(type, offset % target->byte_size()); + } auto st = dynamic_cast(identity); if (!st) { UNEXPECTED; - return nullptr; + return false; } for (auto p = st; p; p = p->getParent()) @@ -132,18 +141,18 @@ const struct_field_info *CheckedStructure::find_field_at_offset_with_type(size_t for (auto field = fields; field->mode != struct_field_info::END; field++) { - if (field->offset > offset) + if (field->mode == struct_field_info::OBJ_METHOD || field->mode == struct_field_info::CLASS_METHOD) continue; - if (field->offset == offset && CheckedStructure(field).identity == type.identity) - return field; + if (field->offset > offset) + continue; - if (auto subfield = CheckedStructure(field).find_field_at_offset_with_type(offset - field->offset, type)) - return subfield; + if (CheckedStructure(field).has_type_at_offset(type, offset - field->offset)) + return true; } } - return nullptr; + return false; } type_identity *Checker::wrap_in_stl_ptr_vector(type_identity *base) From a456b2fcf31cdcafc05a5d1656f2805e7b7f9ca8 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Fri, 13 Mar 2020 00:03:56 -0500 Subject: [PATCH 30/32] fix windows build --- plugins/devel/check-structures-sanity/validate.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/plugins/devel/check-structures-sanity/validate.cpp b/plugins/devel/check-structures-sanity/validate.cpp index a3ed03980..63dc132f4 100644 --- a/plugins/devel/check-structures-sanity/validate.cpp +++ b/plugins/devel/check-structures-sanity/validate.cpp @@ -1,5 +1,12 @@ #include "check-structures-sanity.h" +#ifdef WIN32 +#define WIN32_LEAN_AND_MEAN +#define _WIN32_WINNT 0x0501 +#define WINVER 0x0501 +#include +#endif + bool Checker::is_in_global(const QueueItem & item) { auto fields = df::global::_identity.getFields(); @@ -155,7 +162,7 @@ const char *Checker::get_vtable_name(const QueueItem & item, const CheckedStruct if (!RtlPcToFileHeader(info, &base)) return nullptr; - const char *typeinfo = reinterpret_cast(base) + reinterpret_cast(info)[3]; + const char *typeinfo = reinterpret_cast(base) + reinterpret_cast(info)[3]; const char *name = typeinfo + 16; #else const char *name = reinterpret_cast(info) + 8; From ccf92d4b0ac5089172ca66bb4a048d1ae9369b11 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Fri, 13 Mar 2020 01:09:04 -0500 Subject: [PATCH 31/32] fix check-structures-sanity compile on 64-bit windows --- plugins/devel/check-structures-sanity/validate.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/devel/check-structures-sanity/validate.cpp b/plugins/devel/check-structures-sanity/validate.cpp index 63dc132f4..04ef8422c 100644 --- a/plugins/devel/check-structures-sanity/validate.cpp +++ b/plugins/devel/check-structures-sanity/validate.cpp @@ -159,7 +159,7 @@ const char *Checker::get_vtable_name(const QueueItem & item, const CheckedStruct #ifdef WIN32 #ifdef DFHACK64 void *base; - if (!RtlPcToFileHeader(info, &base)) + if (!RtlPcToFileHeader(const_cast(reinterpret_cast(info)), &base)) return nullptr; const char *typeinfo = reinterpret_cast(base) + reinterpret_cast(info)[3]; From 60722d778fe3054e0c7d7de2ea985b3e85a539da Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Sun, 15 Mar 2020 02:17:19 -0500 Subject: [PATCH 32/32] update structures and scripts --- library/xml | 2 +- scripts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/library/xml b/library/xml index 8ec203292..ad03f7ad7 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit 8ec203292e0aff9dce2a514d4bee01ba722f16de +Subproject commit ad03f7ad753b9a9b22373820e1a41c826c71d63a diff --git a/scripts b/scripts index 54cdfb58b..95a32a0f1 160000 --- a/scripts +++ b/scripts @@ -1 +1 @@ -Subproject commit 54cdfb58b9dd0606fb43117df0a8f5fa5b3a3016 +Subproject commit 95a32a0f1709e17afd927f57675b81effbc42912