Improved Items module and itemhacks plugin

Items module:
* Identified flags for "owned by hostile" and "owned by trader".
* Identified an unknown uint32_t field as being the item age.
* Added and documented ClassNameCheck for more item ref types, though
  they aren't used yet (other than to mark that item ref class as a
  "known" class).
* Added method to get list of an item's unknown item ref classes.
* Made all unknown flags have name 'unk#', instead of some also having
  the name 'u_ngrd#'.

Itemhacks plugin:
* Folded second command into "dumpitems unknown", which will list
  any items which have the unknown vector field non-empty (which seems
  to be associated with tasked items).
* "dumpitems unknown" will also dump any item with a set unknown flag
  or with item refs of an unknown class.

library/include/dfhack/modules/Items.h

@@ -59,40 +59,40 @@ union t_itemflags
     {
         unsigned int on_ground : 1;      ///< 0000 0001 Item on ground
         unsigned int in_job : 1;         ///< 0000 0002 Item currently being used in a job
-        unsigned int u_ngrd1 : 1;        ///< 0000 0004 unknown, unseen
+        unsigned int hostile : 1;        ///< 0000 0004 Item owned by hostile
         unsigned int in_inventory : 1;   ///< 0000 0008 Item in a creature or workshop inventory
 
-        unsigned int u_ngrd2 : 1;        ///< 0000 0010 unknown, lost (artifact)?, unseen
+        unsigned int unk1 : 1;           ///< 0000 0010 unknown, lost (artifact)?, unusable, unseen
         unsigned int in_building : 1;    ///< 0000 0020 Part of a building (including mechanisms, bodies in coffins)
-        unsigned int u_ngrd3 : 1;        ///< 0000 0040 unknown, unseen
+        unsigned int unk2 : 1;           ///< 0000 0040 unknown, unseen
         unsigned int dead_dwarf : 1;     ///< 0000 0080 Dwarf's dead body or body part
 
         unsigned int rotten : 1;         ///< 0000 0100 Rotten food
         unsigned int spider_web : 1;     ///< 0000 0200 Thread in spider web
         unsigned int construction : 1;   ///< 0000 0400 Material used in construction
-        unsigned int u_ngrd5 : 1;        ///< 0000 0800 unknown, unseen
+        unsigned int unk3 : 1;           ///< 0000 0800 unknown, unseen, unusable
 
-        unsigned int unk3 : 1;           ///< 0000 1000 unknown, unseen
-        unsigned int u_ngrd6 : 1;        ///< 0000 2000 unknown, unseen
+        unsigned int unk4 : 1;           ///< 0000 1000 unknown, unseen
+        unsigned int unk5 : 1;           ///< 0000 2000 unknown, unseen
         unsigned int foreign : 1;        ///< 0000 4000 Item is imported
-        unsigned int u_ngrd7 : 1;        ///< 0000 8000 unknown, unseen
+        unsigned int trader : 1;         ///< 0000 8000 Item ownwed by trader
 
         unsigned int owned : 1;          ///< 0001 0000 Item is owned by a dwarf
         unsigned int garbage_colect : 1; ///< 0002 0000 Marked for deallocation by DF it seems
         unsigned int artifact1 : 1;      ///< 0004 0000 Artifact ?
         unsigned int forbid : 1;         ///< 0008 0000 Forbidden item
 
-        unsigned int unk5 : 1;           ///< 0010 0000 unknown, unseen
+        unsigned int unk6 : 1;           ///< 0010 0000 unknown, unseen
         unsigned int dump : 1;           ///< 0020 0000 Designated for dumping
         unsigned int on_fire: 1;         ///< 0040 0000 Indicates if item is on fire, Will Set Item On Fire if Set!
         unsigned int melt : 1;           ///< 0080 0000 Designated for melting, if applicable
 
         unsigned int hidden : 1;       ///< 0100 0000 Hidden item
         unsigned int in_chest : 1;     ///< 0200 0000 Stored in chest/part of well?
-        unsigned int unk6 : 1;         ///< 0400 0000 unknown, unseen
+        unsigned int unk7 : 1;         ///< 0400 0000 unknown, unseen
         unsigned int artifact2 : 1;    ///< 0800 0000 Artifact ?
 
-        unsigned int unk8 : 1;         ///< 1000 0000 unknown, unseen
+        unsigned int unk8 : 1;         ///< 1000 0000 unknown, unseen, common
         unsigned int unk9 : 1;         ///< 2000 0000 unknown, set when viewing details
         unsigned int unk10 : 1;        ///< 4000 0000 unknown, unseen
         unsigned int unk11 : 1;        ///< 8000 0000 unknown, unseen
@@ -120,9 +120,9 @@ struct t_item : public t_virtual
     int16_t z;          // 0x8 + 2
     // 2B padding          0xA + 2
     t_itemflags flags;  // 0xC + 4
-    uint32_t unk1;      // 0x10 + 4
+    uint32_t age ;      // 0x10 + 4
     uint32_t id;        // 0x14 + 4
-    std::vector<void *> unk2;// usage is pretty rare
+    std::vector<void *> unk1;// Used by tasked items.
     std::vector<t_itemref *> itemrefs;
 };
 
@@ -187,7 +187,10 @@ public:
     /// wipe out the owner records
     bool removeItemOwner(dfh_item &item, Creatures *creatures);
 
-    bool readItemRefs(const dfh_item &item, const ClassNameCheck &classname, std::vector<int32_t> &values);
+    bool readItemRefs(const dfh_item &item, const ClassNameCheck &classname,
+                      std::vector<int32_t> &values);
+    bool unknownRefs(const dfh_item &item, std::vector<std::string>& names,
+                     std::vector<int32_t>& values);
 private:
     class Private;
     Private* d;

library/modules/Items.cpp

@@ -30,6 +30,7 @@ distribution.
 #include <vector>
 #include <cstdio>
 #include <map>
+#include <set>
 using namespace std;
 
 #include "dfhack/Types.h"
@@ -427,9 +428,48 @@ class Items::Private
         uint32_t refVectorOffset;
         uint32_t idFieldOffset;
         uint32_t itemVectorAddress;
+
         ClassNameCheck isOwnerRefClass;
         ClassNameCheck isContainerRefClass;
         ClassNameCheck isContainsRefClass;
+
+        // Similar to isOwnerRefClass.  Value is unique to each creature, but
+        // different than the creature's id.
+        ClassNameCheck isUnitHolderRefClass;
+
+        // One of these is present for each creature contained in a cage.
+        // The value is similar to that for isUnitHolderRefClass, different
+        // than the creature's ID but unique for each creature.
+        ClassNameCheck isCagedUnitRefClass;
+
+        // ID of bulding containing/holding the item.
+        ClassNameCheck isBuildingHolderRefClass;
+
+        // Building ID of lever/etc which triggers bridge/etc holding
+        // this mechanism.
+        ClassNameCheck isTriggeredByRefClass;
+
+        // Building ID of bridge/etc which is triggered by lever/etc holding
+        // this mechanism.
+        ClassNameCheck isTriggerTargetRefClass;
+
+        // Civilization ID of owner of item, for items not owned by the
+        // fortress.
+        ClassNameCheck isEntityOwnerRefClass;
+
+        // Item has been offered to the caravan.  The value is the
+        // civilization ID of
+        ClassNameCheck isOfferedRefClass;
+
+        // Item is in a depot for trade.  Purpose of value is unknown, but is
+        // different for each item, even in the same depot at the same time.
+        ClassNameCheck isTradingRefClass;
+
+        // Item is flying or falling through the air.  The value seems to
+        // be the ID for a "projectile information" object.
+        ClassNameCheck isProjectileRefClass;
+
+        std::set<std::string> knownItemRefTypes;
 };
 
 Items::Items()
@@ -438,14 +478,34 @@ Items::Items()
     d = new Private;
     d->owner = c.p;
 
-    d->isOwnerRefClass = ClassNameCheck("general_ref_unit_itemownerst");
-    d->isContainerRefClass = ClassNameCheck("general_ref_contained_in_itemst");
-    d->isContainsRefClass = ClassNameCheck("general_ref_contains_itemst");
-
     DFHack::OffsetGroup* itemGroup = c.vinfo->getGroup("Items");
     d->itemVectorAddress = itemGroup->getAddress("items_vector");
     d->idFieldOffset = itemGroup->getOffset("id");
     d->refVectorOffset = itemGroup->getOffset("item_ref_vector");
+
+    d->isOwnerRefClass = ClassNameCheck("general_ref_unit_itemownerst");
+    d->isContainerRefClass = ClassNameCheck("general_ref_contained_in_itemst");
+    d->isContainsRefClass = ClassNameCheck("general_ref_contains_itemst");
+    d->isUnitHolderRefClass = ClassNameCheck("general_ref_unit_holderst");
+    d->isCagedUnitRefClass = ClassNameCheck("general_ref_contains_unitst");
+    d->isBuildingHolderRefClass
+        = ClassNameCheck("general_ref_building_holderst");
+    d->isTriggeredByRefClass = ClassNameCheck("general_ref_building_triggerst");
+    d->isTriggerTargetRefClass
+        = ClassNameCheck("general_ref_building_triggertargetst");
+    d->isEntityOwnerRefClass = ClassNameCheck("general_ref_entity_itemownerst");
+    d->isOfferedRefClass = ClassNameCheck("general_ref_entity_offeredst");
+    d->isTradingRefClass = ClassNameCheck("general_ref_unit_tradebringerst");
+    d->isProjectileRefClass = ClassNameCheck("general_ref_projectilest");
+
+    std::vector<std::string> known_names;
+    ClassNameCheck::getKnownClassNames(known_names);
+
+    for (size_t i = 0; i < known_names.size(); i++)
+    {
+        if (known_names[i].find("general_ref_") == 0)
+            d->knownItemRefTypes.insert(known_names[i]);
+    }
 }
 
 bool Items::Start()
@@ -562,6 +622,28 @@ bool Items::readItemRefs(const dfh_item &item, const ClassNameCheck &classname,
     return !values.empty();
 }
 
+bool Items::unknownRefs(const dfh_item &item, std::vector<std::string>& names,
+                        std::vector<int32_t>& values)
+{
+    names.clear();
+    values.clear();
+
+    std::vector <t_itemref *> &p_refs = item.base->itemrefs;
+
+    for (uint32_t i=0; i<p_refs.size(); i++)
+    {
+        std::string name = p_refs[i]->getClassName();
+
+        if (d->knownItemRefTypes.find(name) == d->knownItemRefTypes.end())
+        {
+            names.push_back(name);
+            values.push_back(p_refs[i]->value);
+        }
+    }
+
+    return (names.size() > 0);
+}
+
 bool Items::removeItemOwner(dfh_item &item, Creatures *creatures)
 {
     std::vector <t_itemref *> &p_refs = item.base->itemrefs;

plugins/itemhacks.cpp

@@ -6,24 +6,165 @@
 #include <vector>
 #include <string>
 #include <string.h>
+#include <stdio.h> // sprintf()
 
 using std::vector;
 using std::string;
 using namespace DFHack;
 
+//////////////////////
+// START item choosers
+//////////////////////
+
+class item_chooser
+{
+public:
+    item_chooser(Core* _c, DFHack::Items* _Items) : c(_c), Items(_Items)
+    {
+    }
+
+    virtual bool doPrint(DFHack::dfh_item *itm) = 0;
+
+    virtual void postPrint(DFHack::dfh_item *itm)
+    {
+    }
+
+protected:
+    Core          *c;
+    DFHack::Items *Items;
+};
+
+class choose_all : public item_chooser
+{
+public:
+    choose_all(Core* _c, ::Items* _Items) : item_chooser(_c, _Items)
+    {
+    }
+
+    virtual bool doPrint(DFHack::dfh_item *itm)
+    {
+        return (true);
+    }
+
+};
+
+class choose_unknown : public item_chooser
+{
+public:
+    choose_unknown(Core* _c, ::Items* _Items) : item_chooser(_c, _Items)
+    {
+    }
+
+    virtual bool doPrint(DFHack::dfh_item *itm)
+    {
+        if (itm->base->unk1.size() > 0)
+            return true;
+
+        std::vector<std::string> refs;
+        std::vector<int32_t>     values;
+        if (Items->unknownRefs(*itm, refs, values))
+            return true;
+
+        t_itemflags &f = itm->base->flags;
+
+        return (f.unk1 || f.unk2 || f.unk3 || f.unk4 || f.unk5 ||
+                f.unk6 || f.unk7 ||
+                // f.unk8 || f.unk9 || /* Too common */
+                f.unk10 || f.unk11);
+    }
+
+    virtual void postPrint(DFHack::dfh_item *itm) 
+    {
+        std::vector<std::string> flags;
+
+        t_itemflags &f = itm->base->flags;
+
+        if (itm->base->unk1.size() > 0)
+            c->con.print("      vec1: %p\n", itm->base->unk1[0]);
+
+        std::vector<std::string> refs;
+        std::vector<int32_t>     values;
+        if (Items->unknownRefs(*itm, refs, values))
+        {
+            c->con.print("      refs: ");
+            for (size_t i = 0; i < refs.size(); i++)
+            {
+                c->con.print("%s: %d", refs[i].c_str(), values[i]);
+                if ( (i + 1) < refs.size() )
+                    c->con.print(", ");
+            }
+            c->con.print("\n");
+        }
+
+        if (f.unk1) flags.push_back("unk1");
+        if (f.unk2) flags.push_back("unk2");
+        if (f.unk3) flags.push_back("unk3");
+        if (f.unk4) flags.push_back("unk4");
+        if (f.unk5) flags.push_back("unk5");
+        if (f.unk6) flags.push_back("unk6");
+        if (f.unk7) flags.push_back("unk7");
+        if (f.unk8) flags.push_back("unk8");
+        if (f.unk9) flags.push_back("unk9");
+        if (f.unk10) flags.push_back("unk10");
+        if (f.unk11) flags.push_back("unk11");
+
+        if (flags.size() > 0)
+        {
+            c->con.print("      flags: ");
+            for (size_t i = 0; i < flags.size(); i++)
+            {
+                c->con.print("%s", flags[i].c_str());
+                if ( (i + 1) < flags.size() )
+                    c->con.print(", ");
+            }
+            c->con.print("\n");
+        }
+    }
+};
+
+
+class choose_cursor : public item_chooser
+{
+public:
+    choose_cursor(Core* _c, ::Items* _Items, int32_t _x, int32_t _y, int32_t _z)
+        : item_chooser(_c, _Items), x(_x), y(_y), z(_z)
+    {
+    }
+
+    virtual bool doPrint(DFHack::dfh_item *itm)
+    {
+        return (itm->base->x == x && itm->base->y == y && itm->base->z == z
+                && itm->base->flags.on_ground
+                && !itm->base->flags.in_chest
+                && !itm->base->flags.in_inventory
+                && !itm->base->flags.in_building);
+    }
+
+protected:
+    int32_t x, y, z;
+};
+
+
+//////////////////////
+// END item choosers
+//////////////////////
+
 DFhackCExport command_result df_dumpitems (Core * c, vector <string> & parameters);
-DFhackCExport command_result df_itscanvec1 (Core * c, vector <string> & parameters);
 
 DFhackCExport const char * plugin_name ( void )
 {
     return "itemhacks";
 }
 
-DFhackCExport command_result plugin_init ( Core * c, std::vector <PluginCommand> &commands)
+DFhackCExport command_result plugin_init ( Core * c,
+                                           std::vector <PluginCommand> &commands)
 {
     commands.clear();
-    commands.push_back(PluginCommand("dumpitems", "Dump items...", df_dumpitems));
-    commands.push_back(PluginCommand("itscanvec1", "Dump items that have the first vector valid.", df_itscanvec1));
+    commands.push_back(PluginCommand("dumpitems",
+               "Dump items\
+\n              Options:\
+\n                 unkown: Dump items that have anything unknown set",
+               df_dumpitems));
     return CR_OK;
 }
 
@@ -32,31 +173,9 @@ DFhackCExport command_result plugin_shutdown ( Core * c )
     return CR_OK;
 }
 
-DFhackCExport command_result df_itscanvec1 (Core * c, vector <string> & parameters)
-{
-    c->Suspend();
-    DFHack::Items * Items = c->getItems();
-    Items->Start();
-    std::vector<t_item *> p_items;
-    Items->readItemVector(p_items);
-    for(int i = 0; i < p_items.size();i++)
-    {
-        t_item * itm = p_items[i];
-        if(itm->unk2.size())
-        {
-            c->con.print("Found %x, size %d\n",itm,itm->unk2.size());
-        }
-    }
-    c->Resume();
-    return CR_OK;
-}
-
 DFhackCExport command_result df_dumpitems (Core * c, vector <string> & parameters)
 {
     c->Suspend();
-    bool print_hex = false;
-    if(parameters.size() && parameters[0] == "hex")
-        print_hex = true;
     DFHack::Materials * Materials = c->getMaterials();
     Materials->ReadAllMaterials();
 
@@ -72,30 +191,56 @@ DFhackCExport command_result df_dumpitems (Core * c, vector <string> & parameter
     Items->readItemVector(p_items);
     uint32_t size = p_items.size();
 
+    item_chooser *chooser = NULL;
+
+    if (x != -30000)
+        chooser = new choose_cursor(c, Items, x, y, z);
+    else if (parameters.size() == 0)
+        chooser = new choose_all(c, Items);
+    else if (parameters[0] == "unknown")
+        chooser = new choose_unknown(c, Items);
+    else
+    {
+        c->con.printerr("Invalid option: %s\n", parameters[0].c_str());
+        Items->Finish();
+        c->Resume();
+        return CR_FAILURE;
+    }
+
     for(size_t i = 0; i < size; i++)
     {
         DFHack::dfh_item itm;
         memset(&itm, 0, sizeof(DFHack::dfh_item));
         Items->readItem(p_items[i],itm);
 
-        if (x != -30000
-            && !(itm.base->x == x && itm.base->y == y && itm.base->z == z
-                 && itm.base->flags.on_ground
-                 && !itm.base->flags.in_chest
-                 && !itm.base->flags.in_inventory
-                 && !itm.base->flags.in_building))
+        if (!chooser->doPrint(&itm))
             continue;
 
+        // Print something useful, instead of (-30000,-30000,-30000), if
+        // the item isn't on the ground.
+        char location[80];
+        if (itm.base->flags.in_chest)
+            sprintf(location, "chest");
+        else if (itm.base->flags.in_inventory)
+            sprintf(location, "inventory");
+        else if (itm.base->flags.in_building)
+            sprintf(location, "building");
+        else
+            sprintf(location, "%d,%d,%d", itm.base->x, itm.base->y,
+                    itm.base->z);
+
         c->con.print(
-            "%5d: addr:0x%08x %6d %08x (%d,%d,%d) vptr:0x%08x [%d] *%d %s - %s\n",
+            "%5d: addr:0x%08x %6d %08x (%s) vptr:0x%08x [%d]\n"
+            "       *%d %s - %s\n",
             i, itm.base, itm.base->id, itm.base->flags.whole,
-            itm.base->x, itm.base->y, itm.base->z,
+            location,
             itm.base->vptr,
             itm.wear_level,
             itm.quantity,
             Items->getItemClass(itm.matdesc.itemType).c_str(),
             Items->getItemDescription(itm, Materials).c_str()
         );
+        chooser->postPrint(&itm);
     /*
         if (print_hex)
             hexdump(DF,p_items[i],0x300);
@@ -116,5 +261,9 @@ DFhackCExport command_result df_dumpitems (Core * c, vector <string> & parameter
         */
     }
     c->Resume();
+
+    Items->Finish();
+    delete chooser;
+
     return CR_OK;
 }