update check-structures-sanity (part 2 of 2)

develop
Ben Lubar 2020-03-10 23:05:59 -05:00
parent e5de783c58
commit e2138a6cc2
No known key found for this signature in database
GPG Key ID: 92939677AB59EDA4
5 changed files with 674 additions and 81 deletions

@ -15,7 +15,7 @@ using namespace DFHack;
#ifdef WIN32 #ifdef WIN32
#define UNEXPECTED __debugbreak() #define UNEXPECTED __debugbreak()
#else #else
#define UNEXPECTED __asm__ volatile ("int $0x03") #define UNEXPECTED __asm__ volatile ("int $0x03; nop")
#endif #endif
#define PTR_ADD(ptr, offset) reinterpret_cast<const void *>(uintptr_t(ptr) + (offset)) #define PTR_ADD(ptr, offset) reinterpret_cast<const void *>(uintptr_t(ptr) + (offset))
@ -33,14 +33,20 @@ struct CheckedStructure
{ {
type_identity *identity; type_identity *identity;
size_t count; size_t count;
enum_identity *eid;
CheckedStructure(); CheckedStructure();
explicit CheckedStructure(type_identity *, size_t = 0); explicit CheckedStructure(type_identity *, size_t = 0);
CheckedStructure(type_identity *, size_t, enum_identity *);
CheckedStructure(const struct_field_info *); CheckedStructure(const struct_field_info *);
size_t full_size() const; size_t full_size() const;
}; };
#define MIN_SIZE_FOR_SUGGEST 64
extern std::map<size_t, std::vector<std::string>> known_types_by_size;
void build_size_table();
namespace namespace
{ {
template<typename T, bool is_pointer = std::is_pointer<T>::value> template<typename T, bool is_pointer = std::is_pointer<T>::value>
@ -59,7 +65,7 @@ class Checker
{ {
color_ostream & out; color_ostream & out;
std::vector<t_memrange> mapped; std::vector<t_memrange> mapped;
std::map<const void *, CheckedStructure> data; std::map<const void *, std::pair<std::string, CheckedStructure>> data;
std::deque<QueueItem> queue; std::deque<QueueItem> queue;
public: public:
size_t checked_count; size_t checked_count;
@ -72,11 +78,19 @@ public:
bool failfast; bool failfast;
Checker(color_ostream & out); Checker(color_ostream & out);
void queue_item(const QueueItem & item, const CheckedStructure & cs); bool queue_item(const QueueItem & item, const CheckedStructure & cs);
void queue_globals(); void queue_globals();
bool process_queue(); bool process_queue();
bool is_valid_dereference(const QueueItem & item, const CheckedStructure & cs, bool quiet = false); bool is_valid_dereference(const QueueItem & item, const CheckedStructure & cs, size_t size, bool quiet);
inline bool is_valid_dereference(const QueueItem & item, const CheckedStructure & cs, bool quiet = false)
{
return is_valid_dereference(item, cs, cs.full_size(), quiet);
}
inline bool is_valid_dereference(const QueueItem & item, size_t size, bool quiet = false)
{
return is_valid_dereference(item, CheckedStructure(df::identity_traits<void *>::get()), size, quiet);
}
template<typename T> template<typename T>
const T validate_and_dereference(const QueueItem & item, bool quiet = false) const T validate_and_dereference(const QueueItem & item, bool quiet = false)
{ {
@ -87,8 +101,15 @@ public:
return *reinterpret_cast<const T *>(item.ptr); return *reinterpret_cast<const T *>(item.ptr);
} }
int64_t get_int_value(const QueueItem & item, type_identity *type, bool quiet = false);
const char *get_vtable_name(const QueueItem & item, const CheckedStructure & cs, bool quiet = false); const char *get_vtable_name(const QueueItem & item, const CheckedStructure & cs, bool quiet = false);
std::pair<const void *, size_t> validate_vector_size(const QueueItem & item, const CheckedStructure & cs, bool quiet = false); std::pair<const void *, size_t> validate_vector_size(const QueueItem & item, const CheckedStructure & cs, bool quiet = false);
size_t get_allocated_size(const QueueItem & item);
#ifndef WIN32
// this function doesn't make sense on windows, where std::string is not pointer-sized.
const std::string *validate_stl_string_pointer(const void *const*);
#endif
static const char *const *get_enum_item_key(enum_identity *identity, int64_t value);
private: private:
color_ostream & fail(int, const QueueItem &, const CheckedStructure &); color_ostream & fail(int, const QueueItem &, const CheckedStructure &);
@ -106,12 +127,25 @@ private:
void dispatch_class(const QueueItem &, const CheckedStructure &); void dispatch_class(const QueueItem &, const CheckedStructure &);
void dispatch_buffer(const QueueItem &, const CheckedStructure &); void dispatch_buffer(const QueueItem &, const CheckedStructure &);
void dispatch_stl_ptr_vector(const QueueItem &, const CheckedStructure &); void dispatch_stl_ptr_vector(const QueueItem &, const CheckedStructure &);
void dispatch_tagged_union(const QueueItem &, const QueueItem &, const CheckedStructure &, const CheckedStructure &);
void dispatch_tagged_union_vector(const QueueItem &, const QueueItem &, const CheckedStructure &, const CheckedStructure &);
void dispatch_untagged_union(const QueueItem &, const CheckedStructure &); void dispatch_untagged_union(const QueueItem &, const CheckedStructure &);
void check_stl_vector(const QueueItem &, type_identity *); void check_unknown_pointer(const QueueItem &);
void check_stl_vector(const QueueItem &, type_identity *, type_identity *);
void check_stl_string(const QueueItem &);
friend struct CheckedStructure; friend struct CheckedStructure;
static type_identity *wrap_in_pointer(type_identity *); static type_identity *wrap_in_pointer(type_identity *);
static type_identity *wrap_in_stl_ptr_vector(type_identity *); static type_identity *wrap_in_stl_ptr_vector(type_identity *);
}; };
#define FAIL(message) (static_cast<color_ostream &>(fail(__LINE__, item, cs) << message) << COLOR_RESET << std::endl) #define FAIL(message) \
do \
{ \
auto & failstream = fail(__LINE__, item, cs); \
failstream << message; \
failstream << COLOR_RESET << std::endl; \
if (failfast) \
UNEXPECTED; \
} \
while (false)

@ -25,19 +25,25 @@ color_ostream & Checker::fail(int line, const QueueItem & item, const CheckedStr
out << COLOR_YELLOW; out << COLOR_YELLOW;
if (maxerrors && maxerrors != ~size_t(0)) if (maxerrors && maxerrors != ~size_t(0))
maxerrors--; maxerrors--;
if (failfast)
UNEXPECTED;
return out; return out;
} }
void Checker::queue_item(const QueueItem & item, const CheckedStructure & cs) bool Checker::queue_item(const QueueItem & item, const CheckedStructure & cs)
{ {
if (data.count(item.ptr) && data.at(item.ptr).full_size() == cs.full_size()) if (data.count(item.ptr))
{ {
// already checked // already checked
// TODO: make sure types are equal auto existing = data.at(item.ptr);
UNEXPECTED; if (cs.identity != existing.second.identity)
return; {
if (cs.identity->type() == IDTYPE_CLASS && existing.second.identity->type() == IDTYPE_CLASS && get_vtable_name(item, cs, false) && static_cast<virtual_identity *>(cs.identity)->is_instance(static_cast<virtual_ptr>(const_cast<void *>(item.ptr))))
{
return false;
}
FAIL("TODO: handle merging structures: " << data.at(item.ptr).first << " overlaps " << item.path << " (same pointer)");
}
return false;
} }
auto ptr_end = PTR_ADD(item.ptr, cs.full_size()); auto ptr_end = PTR_ADD(item.ptr, cs.full_size());
@ -46,24 +52,42 @@ void Checker::queue_item(const QueueItem & item, const CheckedStructure & cs)
if (prev != data.cbegin()) if (prev != data.cbegin())
{ {
prev--; prev--;
if (uintptr_t(prev->first) + prev->second.full_size() > uintptr_t(item.ptr)) if (uintptr_t(prev->first) + prev->second.second.full_size() > uintptr_t(item.ptr))
{ {
// TODO // placeholder algorithm
if (auto sid = dynamic_cast<struct_identity *>(prev->second.second.identity))
{
auto offset = uintptr_t(item.ptr) - uintptr_t(prev->first);
for (auto field = sid->getFields(); field->mode != struct_field_info::END; field++)
{
if (field->offset == offset)
{
if (field->mode == struct_field_info::SUBSTRUCT && field->type == cs.identity)
{
return false;
}
UNEXPECTED; UNEXPECTED;
} }
} }
}
// TODO
FAIL("TODO: handle merging structures: " << prev->second.first << " overlaps " << item.path << " (backward)");
}
}
auto overlap = data.lower_bound(item.ptr); auto overlap = data.lower_bound(item.ptr);
auto overlap_end = data.lower_bound(ptr_end); auto overlap_end = data.lower_bound(ptr_end);
while (overlap != overlap_end) while (overlap != overlap_end)
{ {
// TODO // TODO
UNEXPECTED; FAIL("TODO: handle merging structures: " << overlap->second.first << " overlaps " << item.path << " (forward)");
overlap++; overlap++;
} }
data[item.ptr] = cs; data[item.ptr] = std::make_pair(item.path, cs);
queue.push_back(item); queue.push_back(item);
return true;
} }
void Checker::queue_globals() void Checker::queue_globals()
@ -89,6 +113,12 @@ void Checker::queue_globals()
continue; continue;
} }
if (!strcmp(field->name, "enabler"))
{
// don't care about libgraphics as we have the source code
continue;
}
queue_item(item, cs); queue_item(item, cs);
} }
} }
@ -110,7 +140,7 @@ bool Checker::process_queue()
return true; return true;
} }
dispatch_item(item, cs->second); dispatch_item(item, cs->second.second);
return true; return true;
} }
@ -194,7 +224,6 @@ void Checker::dispatch_single_item(const QueueItem & item, const CheckedStructur
dispatch_stl_ptr_vector(item, cs); dispatch_stl_ptr_vector(item, cs);
break; break;
case IDTYPE_OPAQUE: case IDTYPE_OPAQUE:
UNEXPECTED;
break; break;
case IDTYPE_UNION: case IDTYPE_UNION:
dispatch_untagged_union(item, cs); dispatch_untagged_union(item, cs);
@ -204,27 +233,78 @@ void Checker::dispatch_single_item(const QueueItem & item, const CheckedStructur
void Checker::dispatch_primitive(const QueueItem & item, const CheckedStructure & cs) void Checker::dispatch_primitive(const QueueItem & item, const CheckedStructure & cs)
{ {
if (cs.identity->isConstructed())
{
if (cs.identity == df::identity_traits<std::string>::get()) if (cs.identity == df::identity_traits<std::string>::get())
{ {
// TODO check std::string check_stl_string(item);
UNEXPECTED;
} }
else else if (cs.identity == df::identity_traits<char *>::get())
{ {
// TODO check c strings
UNEXPECTED; UNEXPECTED;
} }
else if (cs.identity == df::identity_traits<bool>::get())
{
auto val = *reinterpret_cast<const uint8_t *>(item.ptr);
if (val > 1 && val != 0xd2)
{
FAIL("invalid value for bool: " << int(val));
} }
}
// TODO: check primitives else if (auto int_id = dynamic_cast<df::integer_identity_base *>(cs.identity))
{
// TODO check ints?
}
else if (auto float_id = dynamic_cast<df::float_identity_base *>(cs.identity))
{
// TODO check floats?
}
else
{
UNEXPECTED; UNEXPECTED;
}
} }
void Checker::dispatch_pointer(const QueueItem & item, const CheckedStructure & cs) void Checker::dispatch_pointer(const QueueItem & item, const CheckedStructure & cs)
{ {
auto identity = static_cast<pointer_identity *>(cs.identity); auto target_ptr = validate_and_dereference<const void *>(item);
// TODO: check pointer if (!target_ptr)
UNEXPECTED; {
return;
}
#ifdef DFHACK64
if (uintptr_t(target_ptr) == 0xd2d2d2d2d2d2d2d2)
#else
if (uintptr_t(target_ptr) == 0xd2d2d2d2)
#endif
{
return;
}
QueueItem target_item(item.path, target_ptr);
auto target = static_cast<pointer_identity *>(cs.identity)->getTarget();
if (!target)
{
check_unknown_pointer(item);
return;
}
// 256 is an arbitrarily chosen size threshold
if (cs.count || target->byte_size() <= 256)
{
// target is small, or we are inside an array of pointers; handle now
if (queue_item(target_item, CheckedStructure(target)))
{
// we insert it into the queue to make sure we're not stuck in a loop
// get it back out of the queue to prevent the queue growing too big
queue.pop_back();
dispatch_item(target_item, CheckedStructure(target));
}
}
else
{
// target is large and not part of an array; handle later
queue_item(target_item, CheckedStructure(target));
}
} }
void Checker::dispatch_container(const QueueItem & item, const CheckedStructure & cs) void Checker::dispatch_container(const QueueItem & item, const CheckedStructure & cs)
{ {
@ -232,11 +312,7 @@ void Checker::dispatch_container(const QueueItem & item, const CheckedStructure
auto base_container = identity->getFullName(nullptr); auto base_container = identity->getFullName(nullptr);
if (base_container == "vector<void>") if (base_container == "vector<void>")
{ {
if (identity->getIndexEnumType()) check_stl_vector(item, identity->getItemType(), identity->getIndexEnumType());
{
UNEXPECTED;
}
check_stl_vector(item, identity->getItemType());
} }
else if (base_container == "deque<void>") else if (base_container == "deque<void>")
{ {
@ -262,12 +338,10 @@ void Checker::dispatch_bit_container(const QueueItem & item, const CheckedStruct
if (base_container == "BitArray<>") if (base_container == "BitArray<>")
{ {
// TODO: check DF bit array // TODO: check DF bit array
UNEXPECTED;
} }
else if (base_container == "vector<bool>") else if (base_container == "vector<bool>")
{ {
// TODO: check stl bit vector // TODO: check stl bit vector
UNEXPECTED;
} }
else else
{ {
@ -281,8 +355,66 @@ void Checker::dispatch_bitfield(const QueueItem & item, const CheckedStructure &
return; return;
} }
// TODO: check bitfields auto bitfield_type = static_cast<bitfield_identity *>(cs.identity);
uint64_t bitfield_value;
switch (bitfield_type->byte_size())
{
case 1:
bitfield_value = validate_and_dereference<uint8_t>(item);
// don't check for uninitialized; too small to be sure
break;
case 2:
bitfield_value = validate_and_dereference<uint16_t>(item);
if (bitfield_value == 0xd2d2)
{
bitfield_value = 0;
}
break;
case 4:
bitfield_value = validate_and_dereference<uint32_t>(item);
if (bitfield_value == 0xd2d2d2d2)
{
bitfield_value = 0;
}
break;
case 8:
bitfield_value = validate_and_dereference<uint64_t>(item);
if (bitfield_value == 0xd2d2d2d2d2d2d2d2)
{
bitfield_value = 0;
}
break;
default:
UNEXPECTED; UNEXPECTED;
bitfield_value = 0;
break;
}
auto num_bits = bitfield_type->getNumBits();
auto bits = bitfield_type->getBits();
for (int i = 0; i < 64; i++)
{
if (!(num_bits & 1))
{
bitfield_value >>= 1;
continue;
}
bitfield_value >>= 1;
if (i >= num_bits || !bits[i].size)
{
FAIL("bitfield bit " << i << " is out of range");
}
else if (unnamed && bits[i].size > 0 && !bits[i].name)
{
FAIL("bitfield bit " << i << " is unnamed");
}
else if (unnamed && !bits[i + bits[i].size].name)
{
FAIL("bitfield bit " << i << " (part of a field starting at bit " << (i + bits[i].size) << ") is unnamed");
}
}
} }
void Checker::dispatch_enum(const QueueItem & item, const CheckedStructure & cs) void Checker::dispatch_enum(const QueueItem & item, const CheckedStructure & cs)
{ {
@ -291,8 +423,31 @@ void Checker::dispatch_enum(const QueueItem & item, const CheckedStructure & cs)
return; return;
} }
// TODO: check enums auto enum_type = static_cast<enum_identity *>(cs.identity);
UNEXPECTED; auto enum_value = get_int_value(item, enum_type->getBaseType());
if (enum_type->byte_size() == 2 && uint16_t(enum_value) == 0xd2d2)
{
return;
}
else if (enum_type->byte_size() == 4 && uint32_t(enum_value) == 0xd2d2d2d2)
{
return;
}
else if (enum_type->byte_size() == 8 && uint64_t(enum_value) == 0xd2d2d2d2d2d2d2d2)
{
return;
}
auto enum_name = get_enum_item_key(enum_type, enum_value);
if (!enum_name)
{
FAIL("enum value (" << enum_value << ") is out of range");
return;
}
if (unnamed && !*enum_name)
{
FAIL("enum value (" << enum_value << ") is unnamed");
}
} }
void Checker::dispatch_struct(const QueueItem & item, const CheckedStructure & cs) void Checker::dispatch_struct(const QueueItem & item, const CheckedStructure & cs)
{ {
@ -300,6 +455,11 @@ void Checker::dispatch_struct(const QueueItem & item, const CheckedStructure & c
for (auto p = identity; p; p = p->getParent()) for (auto p = identity; p; p = p->getParent())
{ {
auto fields = p->getFields(); auto fields = p->getFields();
if (!fields)
{
continue;
}
for (auto field = fields; field->mode != struct_field_info::END; field++) for (auto field = fields; field->mode != struct_field_info::END; field++)
{ {
dispatch_field(item, cs, fields, field); dispatch_field(item, cs, fields, field);
@ -314,16 +474,28 @@ void Checker::dispatch_field(const QueueItem & item, const CheckedStructure & cs
return; return;
} }
auto field_ptr = PTR_ADD(item.ptr, field->offset);
QueueItem field_item(item, field->name, field_ptr);
CheckedStructure field_cs(field);
auto tag_field = find_union_tag(fields, field); auto tag_field = find_union_tag(fields, field);
if (tag_field) if (tag_field)
{ {
UNEXPECTED; auto tag_ptr = PTR_ADD(item.ptr, tag_field->offset);
QueueItem tag_item(item, tag_field->name, tag_ptr);
CheckedStructure tag_cs(tag_field);
if (tag_cs.identity->isContainer())
{
dispatch_tagged_union_vector(field_item, tag_item, field_cs, tag_cs);
}
else
{
dispatch_tagged_union(field_item, tag_item, field_cs, tag_cs);
}
return; return;
} }
auto field_ptr = PTR_ADD(item.ptr, field->offset); dispatch_item(field_item, field_cs);
CheckedStructure field_cs(field);
dispatch_item(QueueItem(item, field->name, field_ptr), field_cs);
} }
void Checker::dispatch_class(const QueueItem & item, const CheckedStructure & cs) void Checker::dispatch_class(const QueueItem & item, const CheckedStructure & cs)
{ {
@ -336,40 +508,158 @@ void Checker::dispatch_class(const QueueItem & item, const CheckedStructure & cs
auto base_identity = static_cast<virtual_identity *>(cs.identity); auto base_identity = static_cast<virtual_identity *>(cs.identity);
auto vptr = static_cast<virtual_ptr>(const_cast<void *>(item.ptr)); auto vptr = static_cast<virtual_ptr>(const_cast<void *>(item.ptr));
if (!base_identity->is_instance(vptr)) auto identity = virtual_identity::get(vptr);
if (!identity)
{
FAIL("unidentified subclass of " << base_identity->getFullName() << ": " << vtable_name);
return;
}
if (base_identity != identity && !base_identity->is_subclass(identity))
{ {
FAIL("expected subclass of " << base_identity->getFullName() << ", but got " << vtable_name); FAIL("expected subclass of " << base_identity->getFullName() << ", but got " << identity->getFullName());
return; return;
} }
auto identity = virtual_identity::get(vptr); if (data.count(item.ptr) && data.at(item.ptr).first == item.path)
{
// TODO: handle cases where this may overlap later data
data.at(item.ptr).second.identity = identity;
}
dispatch_struct(QueueItem(item.path + "<" + identity->getFullName() + ">", item.ptr), CheckedStructure(identity)); dispatch_struct(QueueItem(item.path + "<" + identity->getFullName() + ">", item.ptr), CheckedStructure(identity));
} }
void Checker::dispatch_buffer(const QueueItem & item, const CheckedStructure & cs) void Checker::dispatch_buffer(const QueueItem & item, const CheckedStructure & cs)
{ {
auto identity = static_cast<container_identity *>(cs.identity); auto identity = static_cast<container_identity *>(cs.identity);
if (identity->getIndexEnumType())
{
UNEXPECTED;
}
auto item_identity = identity->getItemType(); auto item_identity = identity->getItemType();
dispatch_item(item, CheckedStructure(item_identity, identity->byte_size() / item_identity->byte_size())); dispatch_item(item, CheckedStructure(item_identity, identity->byte_size() / item_identity->byte_size(), static_cast<enum_identity *>(identity->getIndexEnumType())));
} }
void Checker::dispatch_stl_ptr_vector(const QueueItem & item, const CheckedStructure & cs) void Checker::dispatch_stl_ptr_vector(const QueueItem & item, const CheckedStructure & cs)
{ {
auto identity = static_cast<container_identity *>(cs.identity); auto identity = static_cast<container_identity *>(cs.identity);
if (identity->getIndexEnumType()) auto ptr_type = wrap_in_pointer(identity->getItemType());
check_stl_vector(item, ptr_type, identity->getIndexEnumType());
}
void Checker::dispatch_tagged_union(const QueueItem & item, const QueueItem & tag_item, const CheckedStructure & cs, const CheckedStructure & tag_cs)
{
if (tag_cs.identity->type() != IDTYPE_ENUM || cs.identity->type() != IDTYPE_UNION)
{
UNEXPECTED;
return;
}
auto tag_identity = static_cast<enum_identity *>(tag_cs.identity);
auto tag_value = get_int_value(tag_item, tag_identity->getBaseType());
auto tag_name = get_enum_item_key(tag_identity, tag_value);
if (!tag_name)
{
FAIL("tagged union tag (accessed as " << tag_item.path << ") value (" << tag_value << ") not defined in enum " << tag_cs.identity->getFullName());
return;
}
if (!*tag_name)
{
FAIL("tagged union tag (accessed as " << tag_item.path << ") value (" << tag_value << ") is unnamed");
return;
}
auto union_type = static_cast<union_identity *>(cs.identity);
for (auto field = union_type->getFields(); field->mode != struct_field_info::END; field++)
{
if (strcmp(field->name, *tag_name))
{
continue;
}
if (field->offset != 0)
{
UNEXPECTED;
}
dispatch_item(QueueItem(item, field->name, item.ptr), field);
return;
}
auto union_data_ptr = reinterpret_cast<const uint8_t *>(item.ptr);
uint8_t padding_byte = *union_data_ptr;
if (padding_byte == 0x00 || padding_byte == 0xd2 || padding_byte == 0xff)
{
bool all_padding = true;
for (size_t i = 0; i < union_type->byte_size(); i++)
{
if (union_data_ptr[i] != padding_byte)
{
all_padding = false;
break;
}
}
// don't ask for fields if it's all padding
if (all_padding)
{
return;
}
}
FAIL("tagged union missing " << *tag_name << " field to match tag (accessed as " << tag_item.path << ") value (" << tag_value << ")");
}
void Checker::dispatch_tagged_union_vector(const QueueItem & item, const QueueItem & tag_item, const CheckedStructure & cs, const CheckedStructure & tag_cs)
{
auto union_container_identity = static_cast<container_identity *>(cs.identity);
CheckedStructure union_item_cs(union_container_identity->getItemType());
if (union_container_identity->type() != IDTYPE_CONTAINER)
{
// assume pointer container
union_item_cs.identity = wrap_in_pointer(union_item_cs.identity);
}
auto tag_container_identity = static_cast<container_identity *>(tag_cs.identity);
auto tag_container_base = tag_container_identity->getFullName(nullptr);
if (tag_container_base == "vector<void>")
{
auto vec_union = validate_vector_size(item, union_item_cs);
CheckedStructure tag_item_cs(tag_container_identity->getItemType());
auto vec_tag = validate_vector_size(tag_item, tag_item_cs);
if (!vec_union.first || !vec_tag.first)
{
// invalid vectors (already warned)
return;
}
if (!vec_union.second && !vec_tag.second)
{
// empty vectors
return;
}
if (vec_union.second != vec_tag.second)
{
FAIL("tagged union vector is " << vec_union.second << " elements, but tag vector (accessed as " << tag_item.path << ") is " << vec_tag.second << " elements");
}
for (size_t i = 0; i < vec_union.second && i < vec_tag.second; i++)
{
dispatch_tagged_union(QueueItem(item, i, vec_union.first), QueueItem(tag_item, i, vec_tag.first), union_item_cs, tag_item_cs);
vec_union.first = PTR_ADD(vec_union.first, union_item_cs.identity->byte_size());
vec_tag.first = PTR_ADD(vec_tag.first, tag_item_cs.identity->byte_size());
}
}
else if (tag_container_base == "vector<bool>")
{
// TODO
UNEXPECTED;
}
else
{ {
UNEXPECTED; UNEXPECTED;
} }
auto ptr_type = wrap_in_pointer(identity->getItemType());
check_stl_vector(item, ptr_type);
} }
void Checker::dispatch_untagged_union(const QueueItem & item, const CheckedStructure & cs) void Checker::dispatch_untagged_union(const QueueItem & item, const CheckedStructure & cs)
{ {
// special case for large_integer weirdness
if (cs.identity == df::identity_traits<df::large_integer>::get()) if (cs.identity == df::identity_traits<df::large_integer>::get())
{ {
// it's 16 bytes on 64-bit linux due to a messy header in libgraphics
// but only the first 8 bytes are ever used
dispatch_primitive(item, CheckedStructure(df::identity_traits<int64_t>::get())); dispatch_primitive(item, CheckedStructure(df::identity_traits<int64_t>::get()));
return; return;
} }
@ -377,13 +667,141 @@ void Checker::dispatch_untagged_union(const QueueItem & item, const CheckedStruc
UNEXPECTED; UNEXPECTED;
} }
void Checker::check_stl_vector(const QueueItem & item, type_identity *item_identity) void Checker::check_unknown_pointer(const QueueItem & item)
{
CheckedStructure cs(df::identity_traits<void *>::get());
if (auto allocated_size = get_allocated_size(item))
{
FAIL("pointer to a block of " << allocated_size << " bytes of allocated memory");
if (allocated_size >= MIN_SIZE_FOR_SUGGEST && known_types_by_size.count(allocated_size))
{
FAIL("known types of this size: " << join_strings(", ", known_types_by_size.at(allocated_size)));
}
// check recursively if it's the right size for a pointer
// or if it starts with what might be a valid pointer
if (allocated_size == sizeof(void *) || (allocated_size > sizeof(void *) && is_valid_dereference(item, cs, true)))
{
QueueItem ptr_item(item, "?ptr?", item.ptr);
if (queue_item(ptr_item, cs))
{
queue.pop_back();
dispatch_pointer(ptr_item, cs);
}
}
}
#ifndef WIN32
else if (auto str = validate_stl_string_pointer(&item.ptr))
{
FAIL("untyped pointer is actually stl-string with value \"" << *str << "\" (length " << str->length() << ")");
}
#endif
else if (auto vtable_name = get_vtable_name(QueueItem(item.path, &item.ptr), cs, true))
{
FAIL("pointer to a vtable: " << vtable_name);
}
else if (sizes)
{
//FAIL("pointer to memory with no size information");
}
}
void Checker::check_stl_vector(const QueueItem & item, type_identity *item_identity, type_identity *eid)
{ {
auto vec_items = validate_vector_size(item, CheckedStructure(item_identity)); auto vec_items = validate_vector_size(item, CheckedStructure(item_identity));
// skip bad pointer vectors
if (item.path.length() > 4 && item.path.substr(item.path.length() - 4) == ".bad" && item_identity->type() == IDTYPE_POINTER)
{
return;
}
if (vec_items.first && vec_items.second) if (vec_items.first && vec_items.second)
{ {
QueueItem items_item(item.path, vec_items.first); QueueItem items_item(item.path, vec_items.first);
CheckedStructure items_cs(item_identity, vec_items.second); CheckedStructure items_cs(item_identity, vec_items.second, static_cast<enum_identity *>(eid));
queue_item(items_item, items_cs); queue_item(items_item, items_cs);
} }
} }
void Checker::check_stl_string(const QueueItem & item)
{
const static CheckedStructure cs(df::identity_traits<std::string>::get());
#ifdef WIN32
struct string_data
{
union
{
uintptr_t start;
char local_data[16];
};
size_t length;
size_t capacity;
};
#else
struct string_data
{
struct string_data_inner
{
size_t length;
size_t capacity;
int32_t refcount;
} *ptr;
};
#endif
auto string = reinterpret_cast<const string_data *>(item.ptr);
#ifdef WIN32
bool is_local = string->capacity < 16;
const char *start = is_local ? &string->local_data[0] : reinterpret_cast<const char *>(string->start);
ptrdiff_t length = string->length;
ptrdiff_t capacity = string->capacity;
#else
if (!is_valid_dereference(QueueItem(item, "?ptr?", string->ptr), 1))
{
// nullptr is NOT okay here
FAIL("invalid string pointer " << stl_sprintf("%p", string->ptr));
return;
}
if (!is_valid_dereference(QueueItem(item, "?hdr?", string->ptr - 1), sizeof(*string->ptr)))
{
return;
}
const char *start = reinterpret_cast<const char *>(string->ptr);
ptrdiff_t length = (string->ptr - 1)->length;
ptrdiff_t capacity = (string->ptr - 1)->capacity;
#endif
if (length < 0)
{
FAIL("string length is negative (" << length << ")");
}
else if (capacity < 0)
{
FAIL("string capacity is negative (" << capacity << ")");
}
else if (capacity < length)
{
FAIL("string capacity (" << capacity << ") is less than length (" << length << ")");
}
#ifndef WIN32
const std::string empty_string;
auto empty_string_data = reinterpret_cast<const string_data *>(&empty_string);
if (sizes && string->ptr != empty_string_data->ptr)
{
size_t allocated_size = get_allocated_size(QueueItem(item, "?hdr?", string->ptr - 1));
size_t expected_size = sizeof(*string->ptr) + capacity + 1;
if (!allocated_size)
{
FAIL("pointer does not appear to be a string");
}
else if (allocated_size != expected_size)
{
FAIL("allocated string data size (" << allocated_size << ") does not match expected size (" << expected_size << ")");
}
}
#endif
}

@ -26,6 +26,9 @@ DFhackCExport command_result plugin_init(color_ostream &, std::vector<PluginComm
"by default, check-structures-sanity reports invalid pointers, vectors, strings, and vtables." "by default, check-structures-sanity reports invalid pointers, vectors, strings, and vtables."
)); ));
known_types_by_size.clear();
build_size_table();
return CR_OK; return CR_OK;
} }

@ -20,7 +20,14 @@ CheckedStructure::CheckedStructure() :
} }
CheckedStructure::CheckedStructure(type_identity *identity, size_t count) : CheckedStructure::CheckedStructure(type_identity *identity, size_t count) :
identity(identity), identity(identity),
count(count) count(count),
eid(nullptr)
{
}
CheckedStructure::CheckedStructure(type_identity *identity, size_t count, enum_identity *eid) :
identity(identity),
count(count),
eid(eid)
{ {
} }
CheckedStructure::CheckedStructure(const struct_field_info *field) : CheckedStructure::CheckedStructure(const struct_field_info *field) :
@ -31,24 +38,8 @@ CheckedStructure::CheckedStructure(const struct_field_info *field) :
UNEXPECTED; UNEXPECTED;
} }
if (field->mode == struct_field_info::STATIC_ARRAY && field->eid)
{
UNEXPECTED;
}
else if (field->type && field->type->isContainer())
{
auto expected_eid = static_cast<container_identity *>(field->type)->getIndexEnumType();
if (field->eid != expected_eid)
{
UNEXPECTED;
}
}
else if (field->eid)
{
UNEXPECTED;
}
identity = field->type; identity = field->type;
eid = field->eid;
switch (field->mode) switch (field->mode)
{ {
case struct_field_info::END: case struct_field_info::END:
@ -129,3 +120,15 @@ type_identity *Checker::wrap_in_pointer(type_identity *base)
{ {
RETURN_CACHED_WRAPPER(df::pointer_identity, base); RETURN_CACHED_WRAPPER(df::pointer_identity, base);
} }
std::map<size_t, std::vector<std::string>> known_types_by_size;
void build_size_table()
{
for (auto & ident : compound_identity::getTopScope())
{
if (ident->byte_size() >= MIN_SIZE_FOR_SUGGEST)
{
known_types_by_size[ident->byte_size()].push_back(ident->getFullName());
}
}
}

@ -1,9 +1,8 @@
#include "check-structures-sanity.h" #include "check-structures-sanity.h"
bool Checker::is_valid_dereference(const QueueItem & item, const CheckedStructure & cs, bool quiet) bool Checker::is_valid_dereference(const QueueItem & item, const CheckedStructure & cs, size_t size, bool quiet)
{ {
auto base = const_cast<void *>(item.ptr); auto base = const_cast<void *>(item.ptr);
auto size = cs.full_size();
if (!base) if (!base)
{ {
// cannot dereference null pointer, but not an error // cannot dereference null pointer, but not an error
@ -83,6 +82,47 @@ bool Checker::is_valid_dereference(const QueueItem & item, const CheckedStructur
#undef FAIL_PTR #undef FAIL_PTR
} }
int64_t Checker::get_int_value(const QueueItem & item, type_identity *type, bool quiet)
{
if (type == df::identity_traits<int32_t>::get())
{
return validate_and_dereference<int32_t>(item, quiet);
}
else if (type == df::identity_traits<uint32_t>::get())
{
return validate_and_dereference<uint32_t>(item, quiet);
}
else if (type == df::identity_traits<int16_t>::get())
{
return validate_and_dereference<int16_t>(item, quiet);
}
else if (type == df::identity_traits<uint16_t>::get())
{
return validate_and_dereference<uint16_t>(item, quiet);
}
else if (type == df::identity_traits<int64_t>::get())
{
return validate_and_dereference<int64_t>(item, quiet);
}
else if (type == df::identity_traits<uint64_t>::get())
{
return int64_t(validate_and_dereference<uint64_t>(item, quiet));
}
else if (type == df::identity_traits<int8_t>::get())
{
return validate_and_dereference<int8_t>(item, quiet);
}
else if (type == df::identity_traits<uint8_t>::get())
{
return validate_and_dereference<uint8_t>(item, quiet);
}
else
{
UNEXPECTED;
return 0;
}
}
const char *Checker::get_vtable_name(const QueueItem & item, const CheckedStructure & cs, bool quiet) const char *Checker::get_vtable_name(const QueueItem & item, const CheckedStructure & cs, bool quiet)
{ {
auto vtable = validate_and_dereference<const void *const*>(QueueItem(item, "?vtable?", item.ptr), quiet); auto vtable = validate_and_dereference<const void *const*>(QueueItem(item, "?vtable?", item.ptr), quiet);
@ -235,3 +275,98 @@ std::pair<const void *, size_t> Checker::validate_vector_size(const QueueItem &
return local_ok ? std::make_pair(start_ptr, ulength / item_size) : std::make_pair(nullptr, 0); return local_ok ? std::make_pair(start_ptr, ulength / item_size) : std::make_pair(nullptr, 0);
} }
size_t Checker::get_allocated_size(const QueueItem & item)
{
if (!sizes)
{
return 0;
}
uint32_t tag = *reinterpret_cast<const uint32_t *>(PTR_ADD(item.ptr, -8));
if (tag == 0xdfdf4ac8)
{
return *reinterpret_cast<const size_t *>(PTR_ADD(item.ptr, -16));
}
return 0;
}
#ifndef WIN32
const std::string *Checker::validate_stl_string_pointer(const void *const* base)
{
std::string empty_string;
if (*base == *reinterpret_cast<void **>(&empty_string))
{
return reinterpret_cast<const std::string *>(base);
}
const struct string_data_inner
{
size_t length;
size_t capacity;
int32_t refcount;
} *str_data = static_cast<const string_data_inner *>(*base) - 1;
uint32_t tag = *reinterpret_cast<const uint32_t *>(PTR_ADD(str_data, -8));
if (tag == 0xdfdf4ac8)
{
size_t allocated_size = *reinterpret_cast<const size_t *>(PTR_ADD(str_data, -16));
size_t expected_size = sizeof(*str_data) + str_data->capacity + 1;
if (allocated_size != expected_size)
{
return nullptr;
}
}
else
{
return nullptr;
}
if (str_data->capacity < str_data->length)
{
return nullptr;
}
const char *ptr = reinterpret_cast<const char *>(*base);
for (size_t i = 0; i < str_data->length; i++)
{
if (!*ptr++)
{
return nullptr;
}
}
if (*ptr++)
{
return nullptr;
}
return reinterpret_cast<const std::string *>(base);
}
#endif
const char *const *Checker::get_enum_item_key(enum_identity *identity, int64_t value)
{
size_t index;
if (auto cplx = identity->getComplex())
{
auto it = cplx->value_index_map.find(value);
if (it == cplx->value_index_map.cend())
{
return nullptr;
}
index = it->second;
}
else
{
if (value < identity->getFirstItem() || value > identity->getLastItem())
{
return nullptr;
}
index = value - identity->getFirstItem();
}
return &identity->getKeys()[index];
}