Merge branch 'develop' of https://github.com/DFHack/dfhack into embark_assistant

develop
PatrikLundell 2020-03-15 17:33:08 +01:00
commit 7e2872e607
24 changed files with 1908 additions and 1436 deletions

@ -79,8 +79,14 @@ set(MAIN_SOURCES
RemoteTools.cpp RemoteTools.cpp
) )
if(WIN32)
set(CONSOLE_SOURCES Console-windows.cpp)
else()
set(CONSOLE_SOURCES Console-posix.cpp)
endif()
set(MAIN_SOURCES_WINDOWS set(MAIN_SOURCES_WINDOWS
Console-windows.cpp ${CONSOLE_SOURCES}
Hooks-windows.cpp Hooks-windows.cpp
PlugLoad-windows.cpp PlugLoad-windows.cpp
Process-windows.cpp Process-windows.cpp
@ -92,21 +98,21 @@ if(WIN32)
endif() endif()
set(MAIN_SOURCES_LINUX set(MAIN_SOURCES_LINUX
Console-posix.cpp ${CONSOLE_SOURCES}
Hooks-linux.cpp Hooks-linux.cpp
PlugLoad-posix.cpp PlugLoad-posix.cpp
Process-linux.cpp Process-linux.cpp
) )
set(MAIN_SOURCES_DARWIN set(MAIN_SOURCES_DARWIN
Console-posix.cpp ${CONSOLE_SOURCES}
Hooks-darwin.cpp Hooks-darwin.cpp
PlugLoad-posix.cpp PlugLoad-posix.cpp
Process-darwin.cpp Process-darwin.cpp
) )
set(MAIN_SOURCES_LINUX_EGGY set(MAIN_SOURCES_LINUX_EGGY
Console-linux.cpp ${CONSOLE_SOURCES}
Hooks-egg.cpp Hooks-egg.cpp
PlugLoad-linux.cpp PlugLoad-linux.cpp
Process-linux.cpp Process-linux.cpp
@ -359,7 +365,7 @@ add_library(dfhack SHARED ${PROJECT_SOURCES})
add_dependencies(dfhack generate_proto_core) add_dependencies(dfhack generate_proto_core)
add_dependencies(dfhack generate_headers) add_dependencies(dfhack generate_headers)
add_library(dfhack-client SHARED RemoteClient.cpp ColorText.cpp MiscUtils.cpp Error.cpp ${PROJECT_PROTO_SRCS}) add_library(dfhack-client SHARED RemoteClient.cpp ColorText.cpp MiscUtils.cpp Error.cpp ${PROJECT_PROTO_SRCS} ${CONSOLE_SOURCES})
add_dependencies(dfhack-client dfhack) add_dependencies(dfhack-client dfhack)
add_executable(dfhack-run dfhack-run.cpp) add_executable(dfhack-run dfhack-run.cpp)

@ -827,21 +827,23 @@ Console::~Console()
delete d; delete d;
} }
bool Console::init(bool sharing) bool Console::init(bool dont_redirect)
{ {
if(sharing)
{
inited = false;
return false;
}
if (!freopen("stdout.log", "w", stdout))
;
d = new Private(); d = new Private();
// make our own weird streams so our IO isn't redirected // make our own weird streams so our IO isn't redirected
d->dfout_C = fopen("/dev/tty", "w"); if (dont_redirect)
{
d->dfout_C = fopen("/dev/stdout", "w");
}
else
{
if (!freopen("stdout.log", "w", stdout))
;
d->dfout_C = fopen("/dev/tty", "w");
}
std::cin.tie(this); std::cin.tie(this);
clear(); clear();
d->supported_terminal = !isUnsupportedTerm() && isatty(STDIN_FILENO); d->supported_terminal = !isUnsupportedTerm() && isatty(STDIN_FILENO);
// init the exit mechanism // init the exit mechanism
if (pipe(d->exit_pipe) == -1) if (pipe(d->exit_pipe) == -1)
; ;
@ -858,6 +860,8 @@ bool Console::shutdown(void)
return true; return true;
lock_guard <recursive_mutex> g(*wlock); lock_guard <recursive_mutex> g(*wlock);
close(d->exit_pipe[1]); close(d->exit_pipe[1]);
if (d->state != Private::con_lineedit)
inited = false;
return true; return true;
} }

@ -1707,7 +1707,6 @@ bool Core::Init()
} }
if (is_text_mode && !is_headless) if (is_text_mode && !is_headless)
{ {
con.init(true);
cerr << "Console is not available. Use dfhack-run to send commands.\n"; cerr << "Console is not available. Use dfhack-run to send commands.\n";
if (!is_text_mode) if (!is_text_mode)
{ {

@ -539,5 +539,14 @@ const struct_field_info *DFHack::find_union_tag(const struct_field_info *fields,
return tag_candidate; return tag_candidate;
} }
auto union_fields = ((struct_identity*)union_field->type)->getFields();
if (tag_container_type->getFullName() == "vector<bool>" &&
union_fields[0].mode != struct_field_info::END &&
union_fields[1].mode != struct_field_info::END &&
union_fields[2].mode == struct_field_info::END)
{
return tag_candidate;
}
return nullptr; return nullptr;
} }

@ -1165,6 +1165,7 @@ static void IndexFields(lua_State *state, int base, struct_identity *pstruct, bo
lua_pop(state, 1); lua_pop(state, 1);
bool add_to_enum = true; bool add_to_enum = true;
const struct_field_info *tag_field = nullptr;
// Handle the field // Handle the field
switch (fields[i].mode) switch (fields[i].mode)
@ -1183,6 +1184,11 @@ static void IndexFields(lua_State *state, int base, struct_identity *pstruct, bo
add_to_enum = false; add_to_enum = false;
break; break;
case struct_field_info::SUBSTRUCT:
case struct_field_info::CONTAINER:
tag_field = find_union_tag(fields, &fields[i]);
break;
default: default:
break; break;
} }
@ -1194,6 +1200,17 @@ static void IndexFields(lua_State *state, int base, struct_identity *pstruct, bo
if (add_to_enum) if (add_to_enum)
AssociateId(state, base+3, ++cnt, name.c_str()); AssociateId(state, base+3, ++cnt, name.c_str());
if (tag_field)
{
// TODO: handle tagged unions
//
// tagged unions are treated as if they have at most one field,
// with the same name as the corresponding enumeration value.
//
// if no field's name matches the enumeration value's name,
// the tagged union is treated as a structure with no fields.
}
lua_pushlightuserdata(state, (void*)&fields[i]); lua_pushlightuserdata(state, (void*)&fields[i]);
lua_setfield(state, base+2, name.c_str()); lua_setfield(state, base+2, name.c_str());
} }

@ -45,6 +45,7 @@ POSSIBILITY OF SUCH DAMAGE.
#include <string> #include <string>
#include <stdint.h> #include <stdint.h>
#include "Console.h"
#include "RemoteClient.h" #include "RemoteClient.h"
#include <cstdio> #include <cstdio>
@ -55,11 +56,10 @@ POSSIBILITY OF SUCH DAMAGE.
using namespace DFHack; using namespace DFHack;
using namespace dfproto; using namespace dfproto;
using std::cout;
int main (int argc, char *argv[]) int main (int argc, char *argv[])
{ {
color_ostream_wrapper out(cout); Console out;
if (argc <= 1) if (argc <= 1)
{ {
@ -85,12 +85,15 @@ int main (int argc, char *argv[])
if (!client.connect()) if (!client.connect())
return 2; return 2;
out.init(true);
command_result rv; command_result rv;
if (strcmp(argv[1], "--lua") == 0) if (strcmp(argv[1], "--lua") == 0)
{ {
if (argc <= 3) if (argc <= 3)
{ {
out.shutdown();
fprintf(stderr, "Usage: dfhack-run --lua <module> <function> [args...]\n"); fprintf(stderr, "Usage: dfhack-run --lua <module> <function> [args...]\n");
return 2; return 2;
} }
@ -99,6 +102,7 @@ int main (int argc, char *argv[])
if (!run_call.bind(&client, "RunLua")) if (!run_call.bind(&client, "RunLua"))
{ {
out.shutdown();
fprintf(stderr, "No RunLua protocol function found."); fprintf(stderr, "No RunLua protocol function found.");
return 3; return 3;
} }
@ -130,6 +134,7 @@ int main (int argc, char *argv[])
} }
out.flush(); out.flush();
out.shutdown();
if (rv != CR_OK) if (rv != CR_OK)
return 1; return 1;

@ -135,7 +135,7 @@ namespace DFHack
///dtor, NOT thread-safe ///dtor, NOT thread-safe
~Console(); ~Console();
/// initialize the console. NOT thread-safe /// initialize the console. NOT thread-safe
bool init( bool sharing ); bool init( bool dont_redirect );
/// shutdown the console. NOT thread-safe /// shutdown the console. NOT thread-safe
bool shutdown( void ); bool shutdown( void );

@ -804,6 +804,9 @@ namespace DFHack {
* *
* If the union field is a container type, the returned tag field is * If the union field is a container type, the returned tag field is
* a container of primitive enum types. * a container of primitive enum types.
*
* As a special case, a container-type union can have a tag field that is
* a bit vector if it has exactly two members.
*/ */
DFHACK_EXPORT const struct_field_info *find_union_tag(const struct_field_info *fields, const struct_field_info *union_field); DFHACK_EXPORT const struct_field_info *find_union_tag(const struct_field_info *fields, const struct_field_info *union_field);
} }

@ -33,6 +33,7 @@ distribution.
#include <stdint.h> #include <stdint.h>
#include "Export.h" #include "Export.h"
#include "Module.h" #include "Module.h"
#include <vector>
namespace DFHack namespace DFHack
{ {
@ -71,8 +72,6 @@ namespace DFHack
class DFHACK_EXPORT Graphic : public Module class DFHACK_EXPORT Graphic : public Module
{ {
public: public:
Graphic();
~Graphic();
bool Finish() bool Finish()
{ {
return true; return true;
@ -82,8 +81,7 @@ namespace DFHack
DFTileSurface* Call(int x, int y); DFTileSurface* Call(int x, int y);
private: private:
struct Private; std::vector<DFTileSurface* (*)(int, int)> funcs;
Private *d;
}; };
} }

@ -1162,7 +1162,7 @@ bool Buildings::deconstruct(df::building *bld)
{ {
auto item = ui_look_list->items[i]; auto item = ui_look_list->items[i];
if (item->type == df::ui_look_list::T_items::Building && if (item->type == df::ui_look_list::T_items::Building &&
item->building == bld) item->data.Building == bld)
{ {
vector_erase_at(ui_look_list->items, i); vector_erase_at(ui_look_list->items, i);
delete item; delete item;

@ -47,40 +47,22 @@ std::unique_ptr<Module> DFHack::createGraphic()
return dts::make_unique<Graphic>(); return dts::make_unique<Graphic>();
} }
struct Graphic::Private
{
bool Started;
vector<DFTileSurface* (*)(int,int)> funcs;
};
Graphic::Graphic()
{
d = new Private;
d->Started = true;
}
Graphic::~Graphic()
{
delete d;
}
bool Graphic::Register(DFTileSurface* (*func)(int,int)) bool Graphic::Register(DFTileSurface* (*func)(int,int))
{ {
d->funcs.push_back(func); funcs.push_back(func);
return true; return true;
} }
bool Graphic::Unregister(DFTileSurface* (*func)(int,int)) bool Graphic::Unregister(DFTileSurface* (*func)(int,int))
{ {
if ( d->funcs.empty() ) return false; if ( funcs.empty() ) return false;
vector<DFTileSurface* (*)(int,int)>::iterator it = d->funcs.begin(); vector<DFTileSurface* (*)(int,int)>::iterator it = funcs.begin();
while ( it != d->funcs.end() ) while ( it != funcs.end() )
{ {
if ( *it == func ) if ( *it == func )
{ {
d->funcs.erase(it); funcs.erase(it);
return true; return true;
} }
it++; it++;
@ -92,12 +74,12 @@ bool Graphic::Unregister(DFTileSurface* (*func)(int,int))
// This will return first DFTileSurface it can get (or NULL if theres none) // This will return first DFTileSurface it can get (or NULL if theres none)
DFTileSurface* Graphic::Call(int x, int y) DFTileSurface* Graphic::Call(int x, int y)
{ {
if ( d->funcs.empty() ) return NULL; if ( funcs.empty() ) return NULL;
DFTileSurface* temp = NULL; DFTileSurface* temp = NULL;
vector<DFTileSurface* (*)(int,int)>::iterator it = d->funcs.begin(); vector<DFTileSurface* (*)(int,int)>::iterator it = funcs.begin();
while ( it != d->funcs.end() ) while ( it != funcs.end() )
{ {
temp = (*it)(x,y); temp = (*it)(x,y);
if ( temp != NULL ) if ( temp != NULL )

@ -1064,12 +1064,12 @@ df::unit *Gui::getAnyUnit(df::viewscreen *top)
if (auto item = vector_get(ui_look_list->items, *ui_look_cursor)) if (auto item = vector_get(ui_look_list->items, *ui_look_cursor))
{ {
if (item->type == df::ui_look_list::T_items::Unit) if (item->type == df::ui_look_list::T_items::Unit)
return item->unit; return item->data.Unit;
else if (item->type == df::ui_look_list::T_items::Item) else if (item->type == df::ui_look_list::T_items::Item)
{ {
if (VIRTUAL_CAST_VAR(corpse, df::item_corpsest, item->item)) if (VIRTUAL_CAST_VAR(corpse, df::item_corpsest, item->data.Item))
return df::unit::find(corpse->unit_id); // loo(k) at corpse return df::unit::find(corpse->unit_id); // loo(k) at corpse
else if (VIRTUAL_CAST_VAR(corpsepiece, df::item_corpsepiecest, item->item)) else if (VIRTUAL_CAST_VAR(corpsepiece, df::item_corpsepiecest, item->data.Item))
return df::unit::find(corpsepiece->unit_id); // loo(k) at corpse piece return df::unit::find(corpsepiece->unit_id); // loo(k) at corpse piece
} }
else if (item->type == df::ui_look_list::T_items::Spatter) else if (item->type == df::ui_look_list::T_items::Spatter)
@ -1201,7 +1201,7 @@ df::item *Gui::getAnyItem(df::viewscreen *top)
auto item = vector_get(ui_look_list->items, *ui_look_cursor); auto item = vector_get(ui_look_list->items, *ui_look_cursor);
if (item && item->type == df::ui_look_list::T_items::Item) if (item && item->type == df::ui_look_list::T_items::Item)
return item->item; return item->data.Item;
else else
return NULL; return NULL;
} }
@ -1266,7 +1266,7 @@ df::building *Gui::getAnyBuilding(df::viewscreen *top)
auto item = vector_get(ui_look_list->items, *ui_look_cursor); auto item = vector_get(ui_look_list->items, *ui_look_cursor);
if (item && item->type == df::ui_look_list::T_items::Building) if (item && item->type == df::ui_look_list::T_items::Building)
return item->building; return item->data.Building;
else else
return NULL; return NULL;
} }

@ -1 +1 @@
Subproject commit 167d54bf7c0e01a71ffb2de548701adc93da9dad Subproject commit ad03f7ad753b9a9b22373820e1a41c826c71d63a

@ -6,7 +6,6 @@ include(FindThreads)
add_definitions(-DDEV_PLUGIN) add_definitions(-DDEV_PLUGIN)
dfhack_plugin(buildprobe buildprobe.cpp) dfhack_plugin(buildprobe buildprobe.cpp)
dfhack_plugin(check-structures-sanity check-structures-sanity.cpp LINK_LIBRARIES lua)
dfhack_plugin(color-dfhack-text color-dfhack-text.cpp) dfhack_plugin(color-dfhack-text color-dfhack-text.cpp)
dfhack_plugin(counters counters.cpp) dfhack_plugin(counters counters.cpp)
dfhack_plugin(dumpmats dumpmats.cpp) dfhack_plugin(dumpmats dumpmats.cpp)
@ -27,3 +26,5 @@ dfhack_plugin(zoom zoom.cpp)
if(UNIX) if(UNIX)
dfhack_plugin(ref-index ref-index.cpp) dfhack_plugin(ref-index ref-index.cpp)
endif() endif()
add_subdirectory(check-structures-sanity)

File diff suppressed because it is too large Load Diff

@ -0,0 +1,8 @@
set(PLUGIN_SRCS
dispatch.cpp
main.cpp
types.cpp
validate.cpp
)
dfhack_plugin(check-structures-sanity ${PLUGIN_SRCS} LINK_LIBRARIES lua)

@ -0,0 +1,159 @@
#pragma once
#include "Console.h"
#include "PluginManager.h"
#include "MemAccess.h"
#include "DataDefs.h"
#include "DataIdentity.h"
#include <deque>
#include <map>
#include <string>
using namespace DFHack;
#ifdef WIN32
#define UNEXPECTED __debugbreak()
#else
#define UNEXPECTED __asm__ volatile ("int $0x03; nop")
#endif
#define PTR_ADD(ptr, offset) reinterpret_cast<const void *>(uintptr_t(ptr) + (offset))
struct QueueItem
{
QueueItem(const std::string &, const void *);
QueueItem(const QueueItem &, const std::string &, const void *);
QueueItem(const QueueItem &, size_t, const void *);
std::string path;
const void *ptr;
};
struct CheckedStructure
{
type_identity *identity;
size_t count;
size_t allocated_count;
enum_identity *eid;
bool ptr_is_array;
bool inside_structure;
CheckedStructure();
explicit CheckedStructure(type_identity *, size_t = 0);
CheckedStructure(type_identity *, size_t, enum_identity *, bool);
CheckedStructure(const struct_field_info *);
size_t full_size() const;
bool has_type_at_offset(const CheckedStructure &, size_t) const;
};
#define MIN_SIZE_FOR_SUGGEST 64
extern std::map<size_t, std::vector<std::string>> known_types_by_size;
void build_size_table();
namespace
{
template<typename T, bool is_pointer = std::is_pointer<T>::value>
struct safe_t
{
typedef T type;
};
template<typename T>
struct safe_t<T, true>
{
typedef void *type;
};
}
class Checker
{
color_ostream & out;
std::vector<t_memrange> mapped;
std::map<const void *, std::pair<std::string, CheckedStructure>> data;
std::deque<QueueItem> queue;
public:
size_t checked_count;
size_t error_count;
size_t maxerrors;
bool maxerrors_reported;
bool enums;
bool sizes;
bool unnamed;
bool failfast;
bool noprogress;
bool maybepointer;
Checker(color_ostream & out);
bool queue_item(const QueueItem & item, CheckedStructure cs);
void queue_globals();
bool process_queue();
bool is_in_global(const QueueItem & item);
bool is_valid_dereference(const QueueItem & item, const CheckedStructure & cs, size_t size, bool quiet);
inline bool is_valid_dereference(const QueueItem & item, const CheckedStructure & cs, bool quiet = false)
{
return is_valid_dereference(item, cs, cs.full_size(), quiet);
}
inline bool is_valid_dereference(const QueueItem & item, size_t size, bool quiet = false)
{
return is_valid_dereference(item, CheckedStructure(df::identity_traits<void *>::get()), size, quiet);
}
template<typename T>
const T validate_and_dereference(const QueueItem & item, bool quiet = false)
{
CheckedStructure cs;
cs.identity = df::identity_traits<typename safe_t<T>::type>::get();
if (!is_valid_dereference(item, cs, quiet))
return T();
return *reinterpret_cast<const T *>(item.ptr);
}
int64_t get_int_value(const QueueItem & item, type_identity *type, bool quiet = false);
const char *get_vtable_name(const QueueItem & item, const CheckedStructure & cs, bool quiet = false);
std::pair<const void *, CheckedStructure> validate_vector_size(const QueueItem & item, const CheckedStructure & cs, bool quiet = false);
size_t get_allocated_size(const QueueItem & item);
#ifndef WIN32
// this function doesn't make sense on windows, where std::string is not pointer-sized.
const std::string *validate_stl_string_pointer(const void *const*);
#endif
static const char *const *get_enum_item_key(enum_identity *identity, int64_t value);
private:
color_ostream & fail(int, const QueueItem &, const CheckedStructure &);
void dispatch_item(const QueueItem &, const CheckedStructure &);
void dispatch_single_item(const QueueItem &, const CheckedStructure &);
void dispatch_primitive(const QueueItem &, const CheckedStructure &);
void dispatch_pointer(const QueueItem &, const CheckedStructure &);
void dispatch_container(const QueueItem &, const CheckedStructure &);
void dispatch_ptr_container(const QueueItem &, const CheckedStructure &);
void dispatch_bit_container(const QueueItem &, const CheckedStructure &);
void dispatch_bitfield(const QueueItem &, const CheckedStructure &);
void dispatch_enum(const QueueItem &, const CheckedStructure &);
void dispatch_struct(const QueueItem &, const CheckedStructure &);
void dispatch_field(const QueueItem &, const CheckedStructure &, const struct_field_info *, const struct_field_info *);
void dispatch_class(const QueueItem &, const CheckedStructure &);
void dispatch_buffer(const QueueItem &, const CheckedStructure &);
void dispatch_stl_ptr_vector(const QueueItem &, const CheckedStructure &);
void dispatch_tagged_union(const QueueItem &, const QueueItem &, const CheckedStructure &, const CheckedStructure &);
void dispatch_tagged_union_vector(const QueueItem &, const QueueItem &, const CheckedStructure &, const CheckedStructure &);
void dispatch_untagged_union(const QueueItem &, const CheckedStructure &);
void check_unknown_pointer(const QueueItem &);
void check_stl_vector(const QueueItem &, type_identity *, type_identity *);
void check_stl_string(const QueueItem &);
void check_possible_pointer(const QueueItem &, const CheckedStructure &);
friend struct CheckedStructure;
static type_identity *wrap_in_pointer(type_identity *);
static type_identity *wrap_in_stl_ptr_vector(type_identity *);
};
#define FAIL(message) \
do \
{ \
auto & failstream = fail(__LINE__, item, cs); \
failstream << message; \
failstream << COLOR_RESET << std::endl; \
if (failfast) \
UNEXPECTED; \
} \
while (false)

@ -0,0 +1,916 @@
#include "check-structures-sanity.h"
#include "df/large_integer.h"
Checker::Checker(color_ostream & out) :
out(out),
checked_count(0),
error_count(0),
maxerrors(~size_t(0)),
maxerrors_reported(false),
enums(false),
sizes(false),
unnamed(false),
failfast(false),
noprogress(!out.is_console()),
maybepointer(false)
{
Core::getInstance().p->getMemRanges(mapped);
}
color_ostream & Checker::fail(int line, const QueueItem & item, const CheckedStructure & cs)
{
error_count++;
out << COLOR_LIGHTRED << "sanity check failed (line " << line << "): ";
out << COLOR_RESET << (cs.identity ? cs.identity->getFullName() : "?");
out << " (accessed as " << item.path << "): ";
out << COLOR_YELLOW;
if (maxerrors && maxerrors != ~size_t(0))
maxerrors--;
return out;
}
bool Checker::queue_item(const QueueItem & item, CheckedStructure cs)
{
if (!cs.identity)
{
UNEXPECTED;
return false;
}
if (cs.identity->type() == IDTYPE_CLASS)
{
if (cs.count)
{
UNEXPECTED;
}
if (get_vtable_name(item, cs, true))
{
auto actual_identity = virtual_identity::get(reinterpret_cast<virtual_ptr>(const_cast<void *>(item.ptr)));
if (static_cast<virtual_identity *>(cs.identity)->is_subclass(actual_identity))
{
cs.identity = actual_identity;
}
}
}
auto ptr_end = PTR_ADD(item.ptr, cs.full_size());
auto prev = data.lower_bound(item.ptr);
if (prev != data.cbegin() && uintptr_t(prev->first) > uintptr_t(item.ptr))
{
prev--;
}
if (prev != data.cend() && uintptr_t(prev->first) <= uintptr_t(item.ptr) && uintptr_t(prev->first) + prev->second.second.full_size() > uintptr_t(item.ptr))
{
auto offset = uintptr_t(item.ptr) - uintptr_t(prev->first);
if (!prev->second.second.has_type_at_offset(cs, offset))
{
// TODO
FAIL("TODO: handle merging structures: " << item.path << " overlaps " << prev->second.first << " (backward)");
return false;
}
// we've already checked this structure, or we're currently queued to do so
return false;
}
auto overlap_start = data.lower_bound(item.ptr);
auto overlap_end = data.lower_bound(ptr_end);
for (auto overlap = overlap_start; overlap != overlap_end; overlap++)
{
auto offset = uintptr_t(overlap->first) - uintptr_t(item.ptr);
if (!cs.has_type_at_offset(overlap->second.second, offset))
{
// TODO
FAIL("TODO: handle merging structures: " << overlap->second.first << " overlaps " << item.path << " (forward)");
return false;
}
}
data.erase(overlap_start, overlap_end);
data[item.ptr] = std::make_pair(item.path, cs);
queue.push_back(item);
return true;
}
void Checker::queue_globals()
{
auto fields = df::global::_identity.getFields();
for (auto field = fields; field->mode != struct_field_info::END; field++)
{
if (!field->offset)
{
UNEXPECTED;
continue;
}
// offset is the position of the DFHack pointer to this global.
auto ptr = *reinterpret_cast<const void **>(field->offset);
QueueItem item(stl_sprintf("df.global.%s", field->name), ptr);
CheckedStructure cs(field);
if (!ptr)
{
FAIL("unknown global address");
continue;
}
if (!strcmp(field->name, "enabler"))
{
// don't care about libgraphics as we have the source code
continue;
}
queue_item(item, cs);
}
}
bool Checker::process_queue()
{
if (queue.empty())
{
return false;
}
auto item = std::move(queue.front());
queue.pop_front();
auto cs = data.find(item.ptr);
if (cs == data.end())
{
// happens if pointer is determined to be part of a larger structure
return true;
}
dispatch_item(item, cs->second.second);
return true;
}
void Checker::dispatch_item(const QueueItem & base, const CheckedStructure & cs)
{
if (!is_valid_dereference(base, cs))
{
return;
}
if (!cs.count)
{
dispatch_single_item(base, cs);
return;
}
if (sizes && !cs.inside_structure)
{
if (auto allocated_size = get_allocated_size(base))
{
auto expected_size = cs.identity->byte_size();
if (cs.allocated_count)
expected_size *= cs.allocated_count;
else if (cs.count)
expected_size *= cs.count;
if (cs.identity->type() == IDTYPE_CLASS && get_vtable_name(base, cs, true))
{
if (cs.count)
{
UNEXPECTED;
}
auto virtual_type = virtual_identity::get(static_cast<virtual_ptr>(const_cast<void *>(base.ptr)));
expected_size = virtual_type->byte_size();
}
auto & item = base;
if (allocated_size > expected_size)
{
FAIL("identified structure is too small (expected " << expected_size << " bytes, but there are " << allocated_size << " bytes allocated)");
}
else if (allocated_size < expected_size)
{
FAIL("identified structure is too big (expected " << expected_size << " bytes, but there are " << allocated_size << " bytes allocated)");
}
}
else
{
UNEXPECTED;
}
}
auto ptr = base.ptr;
auto size = cs.identity->byte_size();
for (size_t i = 0; i < cs.count; i++)
{
QueueItem item(base, i, ptr);
dispatch_single_item(item, cs);
ptr = PTR_ADD(ptr, size);
}
}
void Checker::dispatch_single_item(const QueueItem & item, const CheckedStructure & cs)
{
checked_count++;
if (!maxerrors)
{
if (!maxerrors_reported)
{
FAIL("error limit reached. bailing out with " << (queue.size() + 1) << " items remaining in the queue.");
maxerrors_reported = true;
}
queue.clear();
return;
}
switch (cs.identity->type())
{
case IDTYPE_GLOBAL:
case IDTYPE_FUNCTION:
UNEXPECTED;
break;
case IDTYPE_PRIMITIVE:
dispatch_primitive(item, cs);
break;
case IDTYPE_POINTER:
dispatch_pointer(item, cs);
break;
case IDTYPE_CONTAINER:
dispatch_container(item, cs);
break;
case IDTYPE_PTR_CONTAINER:
dispatch_ptr_container(item, cs);
break;
case IDTYPE_BIT_CONTAINER:
dispatch_bit_container(item, cs);
break;
case IDTYPE_BITFIELD:
dispatch_bitfield(item, cs);
break;
case IDTYPE_ENUM:
dispatch_enum(item, cs);
break;
case IDTYPE_STRUCT:
dispatch_struct(item, cs);
break;
case IDTYPE_CLASS:
dispatch_class(item, cs);
break;
case IDTYPE_BUFFER:
dispatch_buffer(item, cs);
break;
case IDTYPE_STL_PTR_VECTOR:
dispatch_stl_ptr_vector(item, cs);
break;
case IDTYPE_OPAQUE:
break;
case IDTYPE_UNION:
dispatch_untagged_union(item, cs);
break;
}
}
void Checker::dispatch_primitive(const QueueItem & item, const CheckedStructure & cs)
{
if (cs.identity == df::identity_traits<std::string>::get())
{
check_stl_string(item);
}
else if (cs.identity == df::identity_traits<char *>::get())
{
// TODO check c strings
UNEXPECTED;
}
else if (cs.identity == df::identity_traits<bool>::get())
{
auto val = *reinterpret_cast<const uint8_t *>(item.ptr);
if (val > 1 && val != 0xd2)
{
FAIL("invalid value for bool: " << int(val));
}
}
else if (auto int_id = dynamic_cast<df::integer_identity_base *>(cs.identity))
{
check_possible_pointer(item, cs);
// TODO check ints?
}
else if (auto float_id = dynamic_cast<df::float_identity_base *>(cs.identity))
{
// TODO check floats?
}
else
{
UNEXPECTED;
}
}
void Checker::dispatch_pointer(const QueueItem & item, const CheckedStructure & cs)
{
auto target_ptr = validate_and_dereference<const void *>(item);
if (!target_ptr)
{
return;
}
#ifdef DFHACK64
if (uintptr_t(target_ptr) == 0xd2d2d2d2d2d2d2d2)
#else
if (uintptr_t(target_ptr) == 0xd2d2d2d2)
#endif
{
return;
}
QueueItem target_item(item.path, target_ptr);
auto target = static_cast<pointer_identity *>(cs.identity)->getTarget();
if (!target)
{
check_unknown_pointer(target_item);
return;
}
CheckedStructure target_cs(target);
// 256 is an arbitrarily chosen size threshold
if (cs.count || target->byte_size() <= 256)
{
// target is small, or we are inside an array of pointers; handle now
if (queue_item(target_item, target_cs))
{
// we insert it into the queue to make sure we're not stuck in a loop
// get it back out of the queue to prevent the queue growing too big
queue.pop_back();
dispatch_item(target_item, target_cs);
}
}
else
{
// target is large and not part of an array; handle later
queue_item(target_item, target_cs);
}
}
void Checker::dispatch_container(const QueueItem & item, const CheckedStructure & cs)
{
auto identity = static_cast<container_identity *>(cs.identity);
auto base_container = identity->getFullName(nullptr);
if (base_container == "vector<void>")
{
check_stl_vector(item, identity->getItemType(), identity->getIndexEnumType());
}
else if (base_container == "deque<void>")
{
// TODO: check deque?
}
else if (base_container == "DfArray<void>")
{
// TODO: check DfArray
}
else
{
UNEXPECTED;
}
}
void Checker::dispatch_ptr_container(const QueueItem & item, const CheckedStructure & cs)
{
auto identity = static_cast<container_identity *>(cs.identity);
auto base_container = identity->getFullName(nullptr);
{
UNEXPECTED;
}
}
void Checker::dispatch_bit_container(const QueueItem & item, const CheckedStructure & cs)
{
auto identity = static_cast<container_identity *>(cs.identity);
auto base_container = identity->getFullName(nullptr);
if (base_container == "BitArray<>")
{
// TODO: check DF bit array
}
else if (base_container == "vector<bool>")
{
// TODO: check stl bit vector
}
else
{
UNEXPECTED;
}
}
void Checker::dispatch_bitfield(const QueueItem & item, const CheckedStructure & cs)
{
check_possible_pointer(item, cs);
if (!enums)
{
return;
}
auto bitfield_type = static_cast<bitfield_identity *>(cs.identity);
uint64_t bitfield_value;
switch (bitfield_type->byte_size())
{
case 1:
bitfield_value = validate_and_dereference<uint8_t>(item);
// don't check for uninitialized; too small to be sure
break;
case 2:
bitfield_value = validate_and_dereference<uint16_t>(item);
if (bitfield_value == 0xd2d2)
{
bitfield_value = 0;
}
break;
case 4:
bitfield_value = validate_and_dereference<uint32_t>(item);
if (bitfield_value == 0xd2d2d2d2)
{
bitfield_value = 0;
}
break;
case 8:
bitfield_value = validate_and_dereference<uint64_t>(item);
if (bitfield_value == 0xd2d2d2d2d2d2d2d2)
{
bitfield_value = 0;
}
break;
default:
UNEXPECTED;
bitfield_value = 0;
break;
}
auto num_bits = bitfield_type->getNumBits();
auto bits = bitfield_type->getBits();
for (int i = 0; i < 64; i++)
{
if (!(num_bits & 1))
{
bitfield_value >>= 1;
continue;
}
bitfield_value >>= 1;
if (i >= num_bits || !bits[i].size)
{
FAIL("bitfield bit " << i << " is out of range");
}
else if (unnamed && bits[i].size > 0 && !bits[i].name)
{
FAIL("bitfield bit " << i << " is unnamed");
}
else if (unnamed && !bits[i + bits[i].size].name)
{
FAIL("bitfield bit " << i << " (part of a field starting at bit " << (i + bits[i].size) << ") is unnamed");
}
}
}
void Checker::dispatch_enum(const QueueItem & item, const CheckedStructure & cs)
{
check_possible_pointer(item, cs);
if (!enums)
{
return;
}
auto enum_type = static_cast<enum_identity *>(cs.identity);
auto enum_value = get_int_value(item, enum_type->getBaseType());
if (enum_type->byte_size() == 2 && uint16_t(enum_value) == 0xd2d2)
{
return;
}
else if (enum_type->byte_size() == 4 && uint32_t(enum_value) == 0xd2d2d2d2)
{
return;
}
else if (enum_type->byte_size() == 8 && uint64_t(enum_value) == 0xd2d2d2d2d2d2d2d2)
{
return;
}
if (is_in_global(item) && enum_value == 0)
{
return;
}
auto enum_name = get_enum_item_key(enum_type, enum_value);
if (!enum_name)
{
FAIL("enum value (" << enum_value << ") is out of range");
return;
}
if (unnamed && !*enum_name)
{
FAIL("enum value (" << enum_value << ") is unnamed");
}
}
void Checker::dispatch_struct(const QueueItem & item, const CheckedStructure & cs)
{
auto identity = static_cast<struct_identity *>(cs.identity);
for (auto p = identity; p; p = p->getParent())
{
auto fields = p->getFields();
if (!fields)
{
continue;
}
for (auto field = fields; field->mode != struct_field_info::END; field++)
{
dispatch_field(item, cs, fields, field);
}
}
}
void Checker::dispatch_field(const QueueItem & item, const CheckedStructure & cs, const struct_field_info *fields, const struct_field_info *field)
{
if (field->mode == struct_field_info::OBJ_METHOD ||
field->mode == struct_field_info::CLASS_METHOD)
{
return;
}
auto field_ptr = PTR_ADD(item.ptr, field->offset);
QueueItem field_item(item, field->name, field_ptr);
CheckedStructure field_cs(field);
auto tag_field = find_union_tag(fields, field);
if (tag_field)
{
auto tag_ptr = PTR_ADD(item.ptr, tag_field->offset);
QueueItem tag_item(item, tag_field->name, tag_ptr);
CheckedStructure tag_cs(tag_field);
if (tag_cs.identity->isContainer())
{
dispatch_tagged_union_vector(field_item, tag_item, field_cs, tag_cs);
}
else
{
dispatch_tagged_union(field_item, tag_item, field_cs, tag_cs);
}
return;
}
dispatch_item(field_item, field_cs);
}
void Checker::dispatch_class(const QueueItem & item, const CheckedStructure & cs)
{
auto vtable_name = get_vtable_name(item, cs);
if (!vtable_name)
{
// bail out now because virtual_identity::get will crash
return;
}
auto base_identity = static_cast<virtual_identity *>(cs.identity);
auto vptr = static_cast<virtual_ptr>(const_cast<void *>(item.ptr));
auto identity = virtual_identity::get(vptr);
if (!identity)
{
FAIL("unidentified subclass of " << base_identity->getFullName() << ": " << vtable_name);
return;
}
if (base_identity != identity && !base_identity->is_subclass(identity))
{
FAIL("expected subclass of " << base_identity->getFullName() << ", but got " << identity->getFullName());
return;
}
if (data.count(item.ptr) && data.at(item.ptr).first == item.path)
{
// TODO: handle cases where this may overlap later data
data.at(item.ptr).second.identity = identity;
}
dispatch_struct(QueueItem(item.path + "<" + identity->getFullName() + ">", item.ptr), CheckedStructure(identity));
}
void Checker::dispatch_buffer(const QueueItem & item, const CheckedStructure & cs)
{
auto identity = static_cast<container_identity *>(cs.identity);
auto item_identity = identity->getItemType();
dispatch_item(item, CheckedStructure(item_identity, identity->byte_size() / item_identity->byte_size(), static_cast<enum_identity *>(identity->getIndexEnumType()), true));
}
void Checker::dispatch_stl_ptr_vector(const QueueItem & item, const CheckedStructure & cs)
{
auto identity = static_cast<container_identity *>(cs.identity);
auto ptr_type = wrap_in_pointer(identity->getItemType());
check_stl_vector(item, ptr_type, identity->getIndexEnumType());
}
void Checker::dispatch_tagged_union(const QueueItem & item, const QueueItem & tag_item, const CheckedStructure & cs, const CheckedStructure & tag_cs)
{
if (tag_cs.identity->type() != IDTYPE_ENUM || cs.identity->type() != IDTYPE_UNION)
{
UNEXPECTED;
return;
}
auto union_type = static_cast<union_identity *>(cs.identity);
auto union_data_ptr = reinterpret_cast<const uint8_t *>(item.ptr);
uint8_t padding_byte = *union_data_ptr;
bool all_padding = false;
if (padding_byte == 0x00 || padding_byte == 0xd2 || padding_byte == 0xff)
{
all_padding = true;
for (size_t i = 0; i < union_type->byte_size(); i++)
{
if (union_data_ptr[i] != padding_byte)
{
all_padding = false;
break;
}
}
}
auto tag_identity = static_cast<enum_identity *>(tag_cs.identity);
auto tag_value = get_int_value(tag_item, tag_identity->getBaseType());
if (all_padding && padding_byte == 0xd2)
{
// special case: completely uninitialized
switch (tag_identity->byte_size())
{
case 1:
if (tag_value == 0xd2)
{
return;
}
break;
case 2:
if (tag_value == 0xd2d2)
{
return;
}
break;
case 4:
if (tag_value == 0xd2d2d2d2)
{
return;
}
break;
case 8:
if (uint64_t(tag_value) == 0xd2d2d2d2d2d2d2d2)
{
return;
}
break;
default:
UNEXPECTED;
break;
}
}
auto tag_name = get_enum_item_key(tag_identity, tag_value);
if (!tag_name)
{
FAIL("tagged union tag (accessed as " << tag_item.path << ") value (" << tag_value << ") not defined in enum " << tag_cs.identity->getFullName());
return;
}
if (!*tag_name)
{
FAIL("tagged union tag (accessed as " << tag_item.path << ") value (" << tag_value << ") is unnamed");
return;
}
for (auto field = union_type->getFields(); field->mode != struct_field_info::END; field++)
{
if (strcmp(field->name, *tag_name))
{
continue;
}
if (field->offset != 0)
{
UNEXPECTED;
}
dispatch_item(QueueItem(item, field->name, item.ptr), field);
return;
}
// don't ask for fields if it's all padding
if (all_padding)
{
return;
}
FAIL("tagged union missing " << *tag_name << " field to match tag (accessed as " << tag_item.path << ") value (" << tag_value << ")");
}
void Checker::dispatch_tagged_union_vector(const QueueItem & item, const QueueItem & tag_item, const CheckedStructure & cs, const CheckedStructure & tag_cs)
{
auto union_container_identity = static_cast<container_identity *>(cs.identity);
CheckedStructure union_item_cs(union_container_identity->getItemType());
if (union_container_identity->type() != IDTYPE_CONTAINER)
{
// assume pointer container
union_item_cs.identity = wrap_in_pointer(union_item_cs.identity);
}
auto tag_container_identity = static_cast<container_identity *>(tag_cs.identity);
auto tag_container_base = tag_container_identity->getFullName(nullptr);
if (tag_container_base == "vector<void>")
{
auto vec_union = validate_vector_size(item, union_item_cs);
CheckedStructure tag_item_cs(tag_container_identity->getItemType());
auto vec_tag = validate_vector_size(tag_item, tag_item_cs);
if (!vec_union.first || !vec_tag.first)
{
// invalid vectors (already warned)
return;
}
if (!vec_union.second.count && !vec_tag.second.count)
{
// empty vectors
return;
}
if (vec_union.second.count != vec_tag.second.count)
{
FAIL("tagged union vector is " << vec_union.second.count << " elements, but tag vector (accessed as " << tag_item.path << ") is " << vec_tag.second.count << " elements");
}
for (size_t i = 0; i < vec_union.second.count && i < vec_tag.second.count; i++)
{
dispatch_tagged_union(QueueItem(item, i, vec_union.first), QueueItem(tag_item, i, vec_tag.first), union_item_cs, tag_item_cs);
vec_union.first = PTR_ADD(vec_union.first, union_item_cs.identity->byte_size());
vec_tag.first = PTR_ADD(vec_tag.first, tag_item_cs.identity->byte_size());
}
}
else if (tag_container_base == "vector<bool>")
{
// TODO
UNEXPECTED;
}
else
{
UNEXPECTED;
}
}
void Checker::dispatch_untagged_union(const QueueItem & item, const CheckedStructure & cs)
{
// special case for large_integer weirdness
if (cs.identity == df::identity_traits<df::large_integer>::get())
{
// it's 16 bytes on 64-bit linux due to a messy header in libgraphics
// but only the first 8 bytes are ever used
dispatch_primitive(item, CheckedStructure(df::identity_traits<int64_t>::get(), 0, nullptr, cs.inside_structure));
return;
}
UNEXPECTED;
}
void Checker::check_unknown_pointer(const QueueItem & item)
{
const static CheckedStructure cs(nullptr, 0, nullptr, true);
if (auto allocated_size = get_allocated_size(item))
{
FAIL("pointer to a block of " << allocated_size << " bytes of allocated memory");
if (allocated_size >= MIN_SIZE_FOR_SUGGEST && known_types_by_size.count(allocated_size))
{
FAIL("known types of this size: " << join_strings(", ", known_types_by_size.at(allocated_size)));
}
// check recursively if it's the right size for a pointer
// or if it starts with what might be a valid pointer
QueueItem ptr_item(item, "?ptr?", item.ptr);
if (allocated_size == sizeof(void *) || (allocated_size > sizeof(void *) && is_valid_dereference(ptr_item, 1, true)))
{
CheckedStructure ptr_cs(df::identity_traits<void *>::get());
if (queue_item(ptr_item, ptr_cs))
{
queue.pop_back();
dispatch_pointer(ptr_item, ptr_cs);
}
}
}
#ifndef WIN32
else if (auto str = validate_stl_string_pointer(&item.ptr))
{
FAIL("untyped pointer is actually stl-string with value \"" << *str << "\" (length " << str->length() << ")");
}
#endif
else if (auto vtable_name = get_vtable_name(QueueItem(item.path, &item.ptr), cs, true))
{
FAIL("pointer to a vtable: " << vtable_name);
}
else if (sizes)
{
//FAIL("pointer to memory with no size information");
}
}
void Checker::check_stl_vector(const QueueItem & item, type_identity *item_identity, type_identity *eid)
{
auto vec_items = validate_vector_size(item, CheckedStructure(item_identity));
// skip bad pointer vectors
if (item.path.length() > 4 && item.path.substr(item.path.length() - 4) == ".bad" && item_identity->type() == IDTYPE_POINTER)
{
return;
}
if (vec_items.first && vec_items.second.count)
{
QueueItem items_item(item.path, vec_items.first);
queue_item(items_item, vec_items.second);
}
}
void Checker::check_stl_string(const QueueItem & item)
{
const static CheckedStructure cs(df::identity_traits<std::string>::get(), 0, nullptr, true);
#ifdef WIN32
struct string_data
{
union
{
uintptr_t start;
char local_data[16];
};
size_t length;
size_t capacity;
};
#else
struct string_data
{
struct string_data_inner
{
size_t length;
size_t capacity;
int32_t refcount;
} *ptr;
};
#endif
auto string = reinterpret_cast<const string_data *>(item.ptr);
#ifdef WIN32
bool is_local = string->capacity < 16;
const char *start = is_local ? &string->local_data[0] : reinterpret_cast<const char *>(string->start);
ptrdiff_t length = string->length;
ptrdiff_t capacity = string->capacity;
#else
if (!is_valid_dereference(QueueItem(item, "?ptr?", string->ptr), 1))
{
// nullptr is NOT okay here
FAIL("invalid string pointer " << stl_sprintf("%p", string->ptr));
return;
}
if (!is_valid_dereference(QueueItem(item, "?hdr?", string->ptr - 1), sizeof(*string->ptr)))
{
return;
}
const char *start = reinterpret_cast<const char *>(string->ptr);
ptrdiff_t length = (string->ptr - 1)->length;
ptrdiff_t capacity = (string->ptr - 1)->capacity;
#endif
if (length < 0)
{
FAIL("string length is negative (" << length << ")");
}
else if (capacity < 0)
{
FAIL("string capacity is negative (" << capacity << ")");
}
else if (capacity < length)
{
FAIL("string capacity (" << capacity << ") is less than length (" << length << ")");
}
#ifndef WIN32
const std::string empty_string;
auto empty_string_data = reinterpret_cast<const string_data *>(&empty_string);
if (sizes && string->ptr != empty_string_data->ptr)
{
size_t allocated_size = get_allocated_size(QueueItem(item, "?hdr?", string->ptr - 1));
size_t expected_size = sizeof(*string->ptr) + capacity + 1;
if (!allocated_size)
{
FAIL("pointer does not appear to be a string");
}
else if (allocated_size != expected_size)
{
FAIL("allocated string data size (" << allocated_size << ") does not match expected size (" << expected_size << ")");
}
}
#endif
}
void Checker::check_possible_pointer(const QueueItem & item, const CheckedStructure & cs)
{
if (sizes && maybepointer && uintptr_t(item.ptr) % sizeof(void *) == 0)
{
auto ptr = validate_and_dereference<const void *>(item, true);
QueueItem ptr_item(item, "?maybe_pointer?", ptr);
if (ptr && is_valid_dereference(ptr_item, 1, true))
{
check_unknown_pointer(ptr_item);
}
}
}

@ -0,0 +1,133 @@
#include "check-structures-sanity.h"
#include "LuaTools.h"
#include "LuaWrapper.h"
DFHACK_PLUGIN("check-structures-sanity");
static command_result command(color_ostream &, std::vector<std::string> &);
DFhackCExport command_result plugin_init(color_ostream &, std::vector<PluginCommand> & commands)
{
commands.push_back(PluginCommand(
"check-structures-sanity",
"performs a sanity check on df-structures",
command,
false,
"check-structures-sanity [-enums] [-sizes] [-lowmem] [-maxerrors n] [-failfast] [starting_point]\n"
"\n"
"-enums: report unexpected or unnamed enum or bitfield values.\n"
"-sizes: report struct and class sizes that don't match structures. (requires sizecheck)\n"
"-unnamed: report unnamed enum/bitfield values, not just undefined ones.\n"
"-maxerrors n: set the maximum number of errors before bailing out.\n"
"-failfast: crash if any error is encountered. useful only for debugging.\n"
"-maybepointer: report integers that might actually be pointers.\n"
"starting_point: a lua expression or a word like 'screen', 'item', or 'building'. (defaults to df.global)\n"
"\n"
"by default, check-structures-sanity reports invalid pointers, vectors, strings, and vtables."
));
known_types_by_size.clear();
build_size_table();
return CR_OK;
}
static command_result command(color_ostream & out, std::vector<std::string> & parameters)
{
CoreSuspender suspend;
Checker checker(out);
// check parameters with values first
#define VAL_PARAM(name, expr_using_value) \
auto name ## _idx = std::find(parameters.begin(), parameters.end(), "-" #name); \
if (name ## _idx != parameters.end()) \
{ \
if (name ## _idx + 1 == parameters.end()) \
{ \
return CR_WRONG_USAGE; \
} \
try \
{ \
auto value = std::move(*(name ## _idx + 1)); \
parameters.erase((name ## _idx + 1)); \
parameters.erase(name ## _idx); \
checker.name = (expr_using_value); \
} \
catch (std::exception & ex) \
{ \
out.printerr("check-structures-sanity: argument to -%s: %s\n", #name, ex.what()); \
return CR_WRONG_USAGE; \
} \
}
VAL_PARAM(maxerrors, std::stoul(value));
#undef VAL_PARAM
#define BOOL_PARAM(name) \
auto name ## _idx = std::find(parameters.begin(), parameters.end(), "-" #name); \
if (name ## _idx != parameters.end()) \
{ \
checker.name = true; \
parameters.erase(name ## _idx); \
}
BOOL_PARAM(enums);
BOOL_PARAM(sizes);
BOOL_PARAM(unnamed);
BOOL_PARAM(failfast);
BOOL_PARAM(noprogress);
BOOL_PARAM(maybepointer);
#undef BOOL_PARAM
if (parameters.size() > 1)
{
return CR_WRONG_USAGE;
}
if (parameters.empty())
{
checker.queue_globals();
}
else
{
using namespace DFHack::Lua;
using namespace DFHack::Lua::Core;
using namespace DFHack::LuaWrapper;
StackUnwinder unwinder(State);
PushModulePublic(out, "utils", "df_expr_to_ref");
Push(parameters.at(0));
if (!SafeCall(out, 1, 1))
{
return CR_FAILURE;
}
if (!lua_touserdata(State, -1))
{
return CR_WRONG_USAGE;
}
QueueItem item(parameters.at(0), get_object_ref(State, -1));
lua_getfield(State, -1, "_type");
lua_getfield(State, -1, "_identity");
auto identity = reinterpret_cast<type_identity *>(lua_touserdata(State, -1));
if (!identity)
{
out.printerr("could not determine type identity\n");
return CR_FAILURE;
}
checker.queue_item(item, CheckedStructure(identity));
}
while (checker.process_queue())
{
if (!checker.noprogress)
{
out << "checked " << checker.checked_count << " fields\r" << std::flush;
}
}
out << "checked " << checker.checked_count << " fields" << std::endl;
return checker.error_count ? CR_FAILURE : CR_OK;
}

@ -0,0 +1,190 @@
#include "check-structures-sanity.h"
QueueItem::QueueItem(const std::string & path, const void *ptr) :
path(path),
ptr(ptr)
{
}
QueueItem::QueueItem(const QueueItem & parent, const std::string & member, const void *ptr) :
QueueItem(parent.path + "." + member, ptr)
{
}
QueueItem::QueueItem(const QueueItem & parent, size_t index, const void *ptr) :
QueueItem(parent.path + stl_sprintf("[%zu]", index), ptr)
{
}
CheckedStructure::CheckedStructure() :
CheckedStructure(nullptr, 0)
{
}
CheckedStructure::CheckedStructure(type_identity *identity, size_t count) :
CheckedStructure(identity, count, nullptr, false)
{
}
CheckedStructure::CheckedStructure(type_identity *identity, size_t count, enum_identity *eid, bool inside_structure) :
identity(identity),
count(count),
allocated_count(0),
eid(eid),
ptr_is_array(false),
inside_structure(inside_structure)
{
}
CheckedStructure::CheckedStructure(const struct_field_info *field) :
CheckedStructure()
{
if (!field || field->mode == struct_field_info::END)
{
UNEXPECTED;
}
identity = field->type;
eid = field->eid;
inside_structure = true;
switch (field->mode)
{
case struct_field_info::END:
UNEXPECTED;
break;
case struct_field_info::PRIMITIVE:
if (field->count || !field->type)
{
UNEXPECTED;
}
break;
case struct_field_info::STATIC_STRING:
if (!field->count || field->type)
{
UNEXPECTED;
}
identity = df::identity_traits<char>::get();
count = field->count;
break;
case struct_field_info::POINTER:
if (field->count & PTRFLAG_IS_ARRAY)
{
ptr_is_array = true;
}
identity = Checker::wrap_in_pointer(field->type);
break;
case struct_field_info::STATIC_ARRAY:
if (!field->count || !field->type)
{
UNEXPECTED;
}
count = field->count;
break;
case struct_field_info::SUBSTRUCT:
case struct_field_info::CONTAINER:
if (field->count || !field->type)
{
UNEXPECTED;
}
break;
case struct_field_info::STL_VECTOR_PTR:
if (field->count)
{
UNEXPECTED;
}
identity = Checker::wrap_in_stl_ptr_vector(field->type);
break;
case struct_field_info::OBJ_METHOD:
case struct_field_info::CLASS_METHOD:
UNEXPECTED;
break;
}
}
size_t CheckedStructure::full_size() const
{
size_t size = identity->byte_size();
if (count)
{
size *= count;
}
return size;
}
bool CheckedStructure::has_type_at_offset(const CheckedStructure & type, size_t offset) const
{
if (!identity)
return false;
if (offset == 0 && identity == type.identity)
return true;
if (offset >= identity->byte_size() && offset < full_size())
offset %= identity->byte_size();
else if (offset >= identity->byte_size())
return false;
if (identity->type() == IDTYPE_BUFFER)
{
auto target = static_cast<container_identity *>(identity)->getItemType();
return CheckedStructure(target, 0).has_type_at_offset(type, offset % target->byte_size());
}
auto st = dynamic_cast<struct_identity *>(identity);
if (!st)
{
UNEXPECTED;
return false;
}
for (auto p = st; p; p = p->getParent())
{
auto fields = p->getFields();
if (!fields)
continue;
for (auto field = fields; field->mode != struct_field_info::END; field++)
{
if (field->mode == struct_field_info::OBJ_METHOD || field->mode == struct_field_info::CLASS_METHOD)
continue;
if (field->offset > offset)
continue;
if (CheckedStructure(field).has_type_at_offset(type, offset - field->offset))
return true;
}
}
return false;
}
type_identity *Checker::wrap_in_stl_ptr_vector(type_identity *base)
{
static std::map<type_identity *, std::unique_ptr<df::stl_ptr_vector_identity>> wrappers;
auto it = wrappers.find(base);
if (it != wrappers.end())
{
return it->second.get();
}
return (wrappers[base] = dts::make_unique<df::stl_ptr_vector_identity>(base, nullptr)).get();
}
type_identity *Checker::wrap_in_pointer(type_identity *base)
{
static std::map<type_identity *, std::unique_ptr<df::pointer_identity>> wrappers;
auto it = wrappers.find(base);
if (it != wrappers.end())
{
return it->second.get();
}
return (wrappers[base] = dts::make_unique<df::pointer_identity>(base)).get();
}
std::map<size_t, std::vector<std::string>> known_types_by_size;
void build_size_table()
{
for (auto & ident : compound_identity::getTopScope())
{
if (ident->byte_size() >= MIN_SIZE_FOR_SUGGEST)
{
known_types_by_size[ident->byte_size()].push_back(ident->getFullName());
}
}
}

@ -0,0 +1,413 @@
#include "check-structures-sanity.h"
#ifdef WIN32
#define WIN32_LEAN_AND_MEAN
#define _WIN32_WINNT 0x0501
#define WINVER 0x0501
#include <windows.h>
#endif
bool Checker::is_in_global(const QueueItem & item)
{
auto fields = df::global::_identity.getFields();
for (auto field = fields; field->mode != struct_field_info::END; field++)
{
size_t size = CheckedStructure(field).full_size();
auto start = *reinterpret_cast<const void * const*>(field->offset);
auto offset = uintptr_t(item.ptr) - uintptr_t(start);
if (offset < size)
{
return true;
}
}
return false;
}
bool Checker::is_valid_dereference(const QueueItem & item, const CheckedStructure & cs, size_t size, bool quiet)
{
auto base = const_cast<void *>(item.ptr);
if (!base)
{
// cannot dereference null pointer, but not an error
return false;
}
// assumes MALLOC_PERTURB_=45
#ifdef DFHACK64
#define UNINIT_PTR 0xd2d2d2d2d2d2d2d2
#define FAIL_PTR(message) FAIL(stl_sprintf("0x%016zx: ", reinterpret_cast<uintptr_t>(base)) << message)
#else
#define UNINIT_PTR 0xd2d2d2d2
#define FAIL_PTR(message) FAIL(stl_sprintf("0x%08zx: ", reinterpret_cast<uintptr_t>(base)) << message)
#endif
if (uintptr_t(base) == UNINIT_PTR)
{
if (!quiet)
{
FAIL_PTR("uninitialized pointer");
}
return false;
}
bool found = true;
auto expected_start = base;
size_t remaining_size = size;
while (found)
{
found = false;
for (auto & range : mapped)
{
if (!range.isInRange(expected_start))
{
continue;
}
found = true;
if (!range.valid || !range.read)
{
if (!quiet)
{
FAIL_PTR("pointer to invalid memory range");
}
return false;
}
auto expected_end = const_cast<void *>(PTR_ADD(expected_start, remaining_size - 1));
if (size && !range.isInRange(expected_end))
{
auto next_start = PTR_ADD(range.end, 1);
remaining_size -= ptrdiff_t(next_start) - ptrdiff_t(expected_start);
expected_start = const_cast<void *>(next_start);
break;
}
return true;
}
}
if (quiet)
{
return false;
}
if (expected_start == base)
{
FAIL_PTR("pointer not in any mapped range");
}
else
{
FAIL_PTR("pointer exceeds mapped memory bounds (size " << size << ")");
}
return false;
#undef FAIL_PTR
}
int64_t Checker::get_int_value(const QueueItem & item, type_identity *type, bool quiet)
{
if (type == df::identity_traits<int32_t>::get())
{
return validate_and_dereference<int32_t>(item, quiet);
}
else if (type == df::identity_traits<uint32_t>::get())
{
return validate_and_dereference<uint32_t>(item, quiet);
}
else if (type == df::identity_traits<int16_t>::get())
{
return validate_and_dereference<int16_t>(item, quiet);
}
else if (type == df::identity_traits<uint16_t>::get())
{
return validate_and_dereference<uint16_t>(item, quiet);
}
else if (type == df::identity_traits<int64_t>::get())
{
return validate_and_dereference<int64_t>(item, quiet);
}
else if (type == df::identity_traits<uint64_t>::get())
{
return int64_t(validate_and_dereference<uint64_t>(item, quiet));
}
else if (type == df::identity_traits<int8_t>::get())
{
return validate_and_dereference<int8_t>(item, quiet);
}
else if (type == df::identity_traits<uint8_t>::get())
{
return validate_and_dereference<uint8_t>(item, quiet);
}
else
{
UNEXPECTED;
return 0;
}
}
const char *Checker::get_vtable_name(const QueueItem & item, const CheckedStructure & cs, bool quiet)
{
auto vtable = validate_and_dereference<const void *const*>(QueueItem(item, "?vtable?", item.ptr), quiet);
if (!vtable)
return nullptr;
auto info = validate_and_dereference<const char *const*>(QueueItem(item, "?vtable?.info", vtable - 1), quiet);
if (!info)
return nullptr;
#ifdef WIN32
#ifdef DFHACK64
void *base;
if (!RtlPcToFileHeader(const_cast<void *>(reinterpret_cast<const void *>(info)), &base))
return nullptr;
const char *typeinfo = reinterpret_cast<const char *>(base) + reinterpret_cast<const int32_t *>(info)[3];
const char *name = typeinfo + 16;
#else
const char *name = reinterpret_cast<const char *>(info) + 8;
#endif
#else
auto name = validate_and_dereference<const char *>(QueueItem(item, "?vtable?.info.name", info + 1), quiet);
#endif
for (auto & range : mapped)
{
if (!range.isInRange(const_cast<char *>(name)))
{
continue;
}
if (!range.valid || !range.read)
{
if (!quiet)
{
FAIL("pointer to invalid memory range");
}
return nullptr;
}
const char *first_letter = nullptr;
bool letter = false;
for (const char *p = name; ; p++)
{
if (!range.isInRange(const_cast<char *>(p)))
{
return nullptr;
}
if ((*p >= 'a' && *p <= 'z') || *p == '_')
{
if (!letter)
{
first_letter = p;
}
letter = true;
}
else if (!*p)
{
return first_letter;
}
}
}
return nullptr;
}
std::pair<const void *, CheckedStructure> Checker::validate_vector_size(const QueueItem & item, const CheckedStructure & cs, bool quiet)
{
using ret_type = std::pair<const void *, CheckedStructure>;
struct vector_data
{
uintptr_t start;
uintptr_t finish;
uintptr_t end_of_storage;
};
vector_data vector = *reinterpret_cast<const vector_data *>(item.ptr);
ptrdiff_t length = vector.finish - vector.start;
ptrdiff_t capacity = vector.end_of_storage - vector.start;
bool local_ok = true;
auto item_size = cs.identity ? cs.identity->byte_size() : 0;
if (!item_size)
{
item_size = 1;
local_ok = false;
}
if (vector.start > vector.finish)
{
local_ok = false;
if (!quiet)
{
FAIL("vector length is negative (" << (length / ptrdiff_t(item_size)) << ")");
}
}
if (vector.start > vector.end_of_storage)
{
local_ok = false;
if (!quiet)
{
FAIL("vector capacity is negative (" << (capacity / ptrdiff_t(item_size)) << ")");
}
}
else if (vector.finish > vector.end_of_storage)
{
local_ok = false;
if (!quiet)
{
FAIL("vector capacity (" << (capacity / ptrdiff_t(item_size)) << ") is less than its length (" << (length / ptrdiff_t(item_size)) << ")");
}
}
size_t ulength = size_t(length);
size_t ucapacity = size_t(capacity);
if (ulength % item_size != 0)
{
local_ok = false;
if (!quiet)
{
FAIL("vector length is non-integer (" << (ulength / item_size) << " items plus " << (ulength % item_size) << " bytes)");
}
}
if (ucapacity % item_size != 0)
{
local_ok = false;
if (!quiet)
{
FAIL("vector capacity is non-integer (" << (ucapacity / item_size) << " items plus " << (ucapacity % item_size) << " bytes)");
}
}
if (local_ok && capacity && !vector.start)
{
if (!quiet)
{
FAIL("vector has null pointer but capacity " << (capacity / item_size));
}
return ret_type();
}
auto start_ptr = reinterpret_cast<const void *>(vector.start);
if (capacity && !is_valid_dereference(QueueItem(item, "?items?", start_ptr), CheckedStructure(cs.identity, capacity / item_size), quiet))
{
local_ok = false;
}
if (!local_ok)
{
return ret_type();
}
CheckedStructure ret_cs(cs.identity, ulength / item_size);
ret_cs.allocated_count = ucapacity / item_size;
return std::make_pair(start_ptr, ret_cs);
}
size_t Checker::get_allocated_size(const QueueItem & item)
{
if (!sizes)
{
return 0;
}
if (uintptr_t(item.ptr) % 32 != 16)
{
return 0;
}
uint32_t tag = *reinterpret_cast<const uint32_t *>(PTR_ADD(item.ptr, -8));
if (tag == 0xdfdf4ac8)
{
return *reinterpret_cast<const size_t *>(PTR_ADD(item.ptr, -16));
}
return 0;
}
#ifndef WIN32
const std::string *Checker::validate_stl_string_pointer(const void *const* base)
{
std::string empty_string;
if (*base == *reinterpret_cast<void **>(&empty_string))
{
return reinterpret_cast<const std::string *>(base);
}
const struct string_data_inner
{
size_t length;
size_t capacity;
int32_t refcount;
} *str_data = static_cast<const string_data_inner *>(*base) - 1;
if (!is_valid_dereference(QueueItem("str", PTR_ADD(str_data, -16)), 16, true))
{
return nullptr;
}
uint32_t tag = *reinterpret_cast<const uint32_t *>(PTR_ADD(str_data, -8));
if (tag == 0xdfdf4ac8)
{
size_t allocated_size = *reinterpret_cast<const size_t *>(PTR_ADD(str_data, -16));
size_t expected_size = sizeof(*str_data) + str_data->capacity + 1;
if (allocated_size != expected_size)
{
return nullptr;
}
}
else
{
return nullptr;
}
if (str_data->capacity < str_data->length)
{
return nullptr;
}
const char *ptr = reinterpret_cast<const char *>(*base);
for (size_t i = 0; i < str_data->length; i++)
{
if (!*ptr++)
{
return nullptr;
}
}
if (*ptr++)
{
return nullptr;
}
return reinterpret_cast<const std::string *>(base);
}
#endif
const char *const *Checker::get_enum_item_key(enum_identity *identity, int64_t value)
{
size_t index;
if (auto cplx = identity->getComplex())
{
auto it = cplx->value_index_map.find(value);
if (it == cplx->value_index_map.cend())
{
return nullptr;
}
index = it->second;
}
else
{
if (value < identity->getFirstItem() || value > identity->getLastItem())
{
return nullptr;
}
index = value - identity->getFirstItem();
}
return &identity->getKeys()[index];
}

@ -1822,16 +1822,16 @@ public:
switch (element->type) switch (element->type)
{ {
case elt_type::Item: case elt_type::Item:
if (element->item) if (element->data.Item)
desc = Items::getDescription(element->item, 0, true); desc = Items::getDescription(element->data.Item, 0, true);
break; break;
case elt_type::Unit: case elt_type::Unit:
if (element->unit) if (element->data.Unit)
desc = get_unit_description(element->unit); desc = get_unit_description(element->data.Unit);
break; break;
case elt_type::Building: case elt_type::Building:
if (element->building) if (element->data.Building)
element->building->getName(&desc); element->data.Building->getName(&desc);
break; break;
default: default:
break; break;

@ -61,7 +61,7 @@ public class find_df_globals extends GhidraScript {
dataAddr = globalAddr.getNewAddress(mem.getInt(globalAddr.add(globalCount * ptrSize * 2 + ptrSize))); dataAddr = globalAddr.getNewAddress(mem.getInt(globalAddr.add(globalCount * ptrSize * 2 + ptrSize)));
} }
String name = StringDataInstance.getStringDataInstance(currentProgram.getListing().createData(nameAddr, TerminatedStringDataType.dataType)).getStringValue(); String name = StringDataInstance.getStringDataInstance(DataUtilities.createData(currentProgram, nameAddr, TerminatedStringDataType.dataType, 0, false, DataUtilities.ClearDataMode.CLEAR_ALL_CONFLICT_DATA)).getStringValue();
createLabel(dataAddr, name, true); createLabel(dataAddr, name, true);
} }

@ -1 +1 @@
Subproject commit 7bab11642bee7a3aa05d69332466f2ea5eaa1a2d Subproject commit 95a32a0f1709e17afd927f57675b81effbc42912