Segmented finder gets backwards pointer scan and arbitrary data search.

data/Memory-ng.xml

@@ -2212,6 +2212,10 @@
             </Group>
             -->
         </Offsets>
+        
+    </Version>
+    <Version name="v0.31.16 linux" os="linux" base="DF2010">
+        <MD5 value="9cca2fa5da509e2f9a1042ddd1f9669c" />
     </Version>
 </DFHack>
 

tools/playground/SegmentedFinder.h

@@ -1,5 +1,8 @@
 #ifndef SEGMENTED_FINDER_H
 #define SEGMENTED_FINDER_H
+#include <malloc.h>
+#include <iosfwd>
+#include <iterator>
 
 class SegmentedFinder;
 class SegmentFinder
@@ -286,15 +289,205 @@ bool vectorAll (SegmentedFinder* s, vecTriplet *x, int )
     return false;
 }
 
-struct Bytestream
+class Bytestreamdata
 {
-    uint32_t length;
+    public:
         void * object;
+        uint32_t length;
+        uint32_t allocated;
+        uint32_t n_used;
+};
+
+class Bytestream
+{
+public:
+    Bytestream(void * obj, uint32_t len, bool alloc = false)
+    {
+        d = new Bytestreamdata();
+        d->allocated = alloc;
+        d->object = obj;
+        d->length = len;
+        d->n_used = 1;
+        constant = false;
+    }
+    Bytestream()
+    {
+        d = new Bytestreamdata();
+        d->allocated = false;
+        d->object = 0;
+        d->length = 0;
+        d->n_used = 1;
+        constant = false;
+    }
+    Bytestream( Bytestream & bs)
+    {
+        d =bs.d;
+        d->n_used++;
+        constant = false;
+    }
+    Bytestream( const Bytestream & bs)
+    {
+        d =bs.d;
+        d->n_used++;
+        constant = true;
+    }
+    ~Bytestream()
+    {
+        d->n_used --;
+        if(d->allocated && d->object && d->n_used == 0)
+        {
+            free (d->object);
+            free (d);
+        }
+    }
+    bool Allocate(size_t bytes)
+    {
+        if(constant)
+            return false;
+        if(d->allocated)
+        {
+            d->object = realloc(d->object, bytes);
+        }
+        else
+        {
+            d->object = malloc( bytes );
+        }
+
+        if(d->object)
+        {
+            d->allocated = bytes;
+            return true;
+        }
+        else
+        {
+            d->allocated = 0;
+            return false;
+        }
+    }
+    bool insert( char what )
+    {
+        if(constant)
+            return false;
+        if(d->length+1 == d->allocated)
+            Allocate(d->allocated * 2);
+        ((char *) d->object)[d->length] = what;
+        d->length ++;
+    }
+    Bytestreamdata * d;
+    bool constant;
 };
+std::ostream& operator<< ( std::ostream& out, Bytestream& bs )
+{
+    if(bs.d->object)
+    {
+        out << "bytestream " << dec << bs.d->length << "/" << bs.d->allocated << " bytes" << endl;
+        for(int i = 0; i < bs.d->length; i++)
+        {
+            out << hex << (int) ((uint8_t *) bs.d->object)[i] << " ";
+        }
+        out << endl;
+    }
+    else
+    {
+        out << "empty bytestresm" << endl;
+    }
+    return out;
+}
+
+std::istream& operator>> ( std::istream& out, Bytestream& bs )
+{
+    string read;
+    while(!out.eof())
+    {
+        string tmp;
+        out >> tmp;
+        read.append(tmp);
+    }
+    cout << read << endl;
+    bs.d->length = 0;
+    size_t first = read.find_first_of("\"");
+    size_t last = read.find_last_of("\"");
+    size_t start = first + 1;
+    if(first == read.npos)
+    {
+        std::transform(read.begin(), read.end(), read.begin(), (int(*)(int)) tolower);
+        bs.Allocate(read.size()); // overkill. size / 2 should be good, but this is safe
+        int state = 0;
+        char big = 0;
+        char small = 0;
+        string::iterator it = read.begin();
+        // iterate through string, construct a bytestream out of 00-FF bytes
+        while(it != read.end())
+        {
+            char reads = *it;
+            if((reads >='0' && reads <= '9'))
+            {
+                if(state == 0)
+                {
+                    big = reads - '0';
+                    state = 1;
+                }
+                else if(state == 1)
+                {
+                    small = reads - '0';
+                    state = 0;
+                    bs.insert(big*16 + small);
+                }
+            }
+            if((reads >= 'a' && reads <= 'f'))
+            {
+                if(state == 0)
+                {
+                    big = reads - 'a' + 10;
+                    state = 1;
+                }
+                else if(state == 1)
+                {
+                    small = reads - 'a' + 10;
+                    state = 0;
+                    bs.insert(big*16 + small);
+                }
+            }
+            it++;
+        }
+        // we end in state= 1. should we add or should we trim... or throw errors?
+        // I decided on adding
+        if (state == 1)
+        {
+            small = 0;
+            bs.insert(big*16 + small);
+        }
+    }
+    else
+    {
+        if(last == first)
+        {
+            // only one "
+            last = read.size();
+        }
+        size_t length = last - start;
+        // construct bytestream out of stuff between ""
+        bs.d->length = length;
+        if(length)
+        {
+            // todo: Bytestream should be able to handle this without external code
+            bs.Allocate(length);
+            bs.d->length = length;
+            const char* strstart = read.c_str();
+            memcpy(bs.d->object, strstart + start, length);
+        }
+        else
+        {
+            bs.d->object = 0;
+        }
+    }
+    cout << bs;
+    return out;
+}
 
 bool findBytestream (SegmentedFinder* s, void *addr, Bytestream compare )
 {
-    if(memcmp(addr, compare.object, compare.length) == 0)
+    if(memcmp(addr, compare.d->object, compare.d->length) == 0)
         return true;
     return false;
 }

tools/playground/incrementalsearch.cpp

@@ -4,6 +4,7 @@
 #include <climits>
 #include <vector>
 #include <map>
+#include <set>
 #include <ctime>
 #include <string.h>
 #include <stdio.h>
@@ -273,7 +274,7 @@ bool getString (string prompt, string & output)
 
 template <class T>
 bool Incremental ( vector <uint64_t> &found, const char * what, T& output,
-                   const char *singular = "address", const char *plural = "addresses" )
+                   const char *singular = "address", const char *plural = "addresses", bool numberz = false )
 {
     string select;
     if(found.empty())
@@ -316,6 +317,8 @@ bool Incremental ( vector <uint64_t> &found, const char * what, T& output,
         return false;
     }
     else
+    {
+        if(numberz)
         {
             if( sscanf(select.c_str(),"0x%x", &output) == 1 )
             {
@@ -327,10 +330,11 @@ bool Incremental ( vector <uint64_t> &found, const char * what, T& output,
                 //cout << dec << output << endl;
                 return true;
             }
-
+        }
         stringstream ss (stringstream::in | stringstream::out);
         ss << select;
         ss >> output;
+        cout << output;
         if(!ss.fail())
         {
             return true;
@@ -358,7 +362,7 @@ void FindIntegers(DFHack::ContextManager & DFMgr, vector <DFHack::t_memrange>& r
     uint32_t test1;
     vector <uint64_t> found;
     found.reserve(100);
-    while(Incremental(found, "integer",test1))
+    while(Incremental(found, "integer",test1,"address", "addresses",true))
     {
         DFMgr.Refresh();
         DFHack::Context * DF = DFMgr.getSingleContext();
@@ -514,6 +518,156 @@ void FindStrings(DFHack::ContextManager & DFMgr, vector <DFHack::t_memrange>& ra
     }
 }
 
+void FindData(DFHack::ContextManager & DFMgr, vector <DFHack::t_memrange>& ranges)
+{
+    vector <uint64_t> found;
+    Bytestream select;
+    while (Incremental(found,"byte stream",select,"byte stream","byte streams"))
+    {
+        DFMgr.Refresh();
+        DFHack::Context * DF = DFMgr.getSingleContext();
+        DF->Attach();
+        SegmentedFinder sf(ranges,DF);
+        sf.Incremental< Bytestream ,uint32_t>(select,1,found, findBytestream);
+        DF->Detach();
+    }
+}
+/*
+    while(Incremental(found, "integer",test1))
+    {
+        DFMgr.Refresh();
+        DFHack::Context * DF = DFMgr.getSingleContext();
+        DF->Attach();
+        SegmentedFinder sf(ranges,DF);
+        switch(size)
+        {
+            case 1:
+                sf.Incremental<uint8_t,uint8_t>(test1,alignment,found, equalityP<uint8_t>);
+                break;
+            case 2:
+                sf.Incremental<uint16_t,uint16_t>(test1,alignment,found, equalityP<uint16_t>);
+                break;
+            case 4:
+                sf.Incremental<uint32_t,uint32_t>(test1,alignment,found, equalityP<uint32_t>);
+                break;
+        }
+        DF->Detach();
+    }
+}
+*/
+void PtrTrace(DFHack::ContextManager & DFMgr, vector <DFHack::t_memrange>& ranges)
+{
+    int element_size;
+    do
+    {
+        getNumber("Set search granularity",element_size, 4);
+    } while (element_size < 1);
+
+    vector <uint64_t> found;
+    set <uint64_t> check; // to detect circles
+    uint32_t select;
+    while (Incremental(found,"address",select,"addresses","addresses",true))
+    {
+        DFMgr.Refresh();
+        found.clear();
+        DFHack::Context * DF = DFMgr.getSingleContext();
+        DF->Attach();
+        SegmentedFinder sf(ranges,DF);
+        cout <<"Starting: 0x" << hex << select << endl;
+        while(sf.getSegmentForAddress(select))
+        {
+            sf.Incremental<uint32_t,uint32_t>(select,element_size,found, equalityP<uint32_t>);
+            if(found.empty())
+            {
+                cout << ".";
+                cout.flush();
+                select -=element_size;
+                continue;
+            }
+            cout << endl;
+            cout <<"Object start: 0x" << hex << select << endl;
+            cout <<"Pointer:      0x" << hex << found[0] << endl;
+            // make sure we don't go in circles'
+            if(check.count(select))
+            {
+                break;
+            }
+            check.insert(select);
+            // ascend
+            select = found[0];
+        }
+        DF->Detach();
+    }
+}
+/*
+{
+    vector <uint64_t> found;
+    Bytestream select;
+    while (Incremental(found,"byte stream",select,"byte stream","byte streams"))
+    {
+        DFMgr.Refresh();
+        DFHack::Context * DF = DFMgr.getSingleContext();
+        DF->Attach();
+        SegmentedFinder sf(ranges,DF);
+        sf.Incremental< Bytestream ,uint32_t>(select,1,found, findBytestream);
+        DF->Detach();
+    }
+}
+*/
+void DataPtrTrace(DFHack::ContextManager & DFMgr, vector <DFHack::t_memrange>& ranges)
+{
+    int element_size;
+    do
+    {
+        getNumber("Set search granularity",element_size, 4);
+    } while (element_size < 1);
+
+    vector <uint64_t> found;
+    set <uint64_t> check; // to detect circles
+    uint32_t select;
+    Bytestream bs_select;
+    DFHack::Context * DF = DFMgr.getSingleContext();
+    DF->Attach();
+    DFMgr.Refresh();
+    found.clear();
+    SegmentedFinder sf(ranges,DF);
+    while(found.empty())
+    {
+        Incremental(found,"byte stream",bs_select,"byte stream","byte streams");
+        
+        sf.Incremental< Bytestream ,uint32_t>(bs_select,1,found, findBytestream);
+    }
+    select = found[0];
+    
+    
+    
+    
+    cout <<"Starting: 0x" << hex << select << endl;
+    while(sf.getSegmentForAddress(select))
+    {
+        sf.Incremental<uint32_t,uint32_t>(select,element_size,found, equalityP<uint32_t>);
+        if(found.empty())
+        {
+            cout << ".";
+            cout.flush();
+            select -=element_size;
+            continue;
+        }
+        cout << endl;
+        cout <<"Object start: 0x" << hex << select << endl;
+        cout <<"Pointer:      0x" << hex << found[0] << endl;
+        // make sure we don't go in circles'
+        if(check.count(select))
+        {
+            break;
+        }
+        check.insert(select);
+        // ascend
+        select = found[0];
+    }
+    DF->Detach();
+}
+
 void printFound(vector <uint64_t> &found, const char * what)
 {
     cout << what << ":" << endl;
@@ -726,7 +880,8 @@ void automatedLangtables(DFHack::Context * DF, vector <DFHack::t_memrange>& rang
         uint64_t Eoffset;
         cout << "Elephant: 0x" << hex << elephant << endl;
         cout << "Elephant: rawname = 0x0" << endl;
-        Eoffset = sf.FindInRange<uint8_t,uint8_t> ('E',equalityP<uint8_t>, elephant, 0x300 );
+        uint8_t letter_E = 'E';
+        Eoffset = sf.FindInRange<uint8_t,uint8_t> (letter_E,equalityP<uint8_t>, elephant, 0x300 );
         if(Eoffset)
         {
             cout << "Elephant: big E = 0x" << hex << Eoffset - elephant << endl;
@@ -742,7 +897,8 @@ void automatedLangtables(DFHack::Context * DF, vector <DFHack::t_memrange>& rang
             cout << "Elephant: extract? vector = 0x" << hex << Eoffset - elephant << endl;
         }
         tilecolors eletc = {7,0,0};
-        Bytestream bs_eletc = {sizeof(tilecolors), &eletc};
+        Bytestream bs_eletc(&eletc, sizeof(tilecolors));
+        cout << bs_eletc;
         Eoffset = sf.FindInRange<Bytestream,tilecolors> (bs_eletc, findBytestream, elephant, 0x300 );
         if(Eoffset)
         {
@@ -775,7 +931,7 @@ void automatedLangtables(DFHack::Context * DF, vector <DFHack::t_memrange>& rang
             cout << "Toad: extract? vector = 0x" << hex << Eoffset - toad << endl;
         }
         tilecolors toadtc = {2,0,0};
-        Bytestream bs_toadc = {sizeof(tilecolors), &toadtc};
+        Bytestream bs_toadc(&toadtc, sizeof(tilecolors));
         Eoffset = sf.FindInRange<Bytestream,tilecolors> (bs_toadc, findBytestream, toad, 0x300 );
         if(Eoffset)
         {
@@ -816,12 +972,12 @@ int main (void)
     "Select search type: 1=number(default), 2=vector by length, 3=vector>object>string,\n"
     "                    4=string, 5=automated offset search, 6=vector by address in its array,\n"
     "                    7=pointer vector by address of an object, 8=vector>first object>string\n"
-    "                    9=string buffers\n";
+    "                    9=string buffers, 10=known data, 11=backpointers, 12=data+backpointers\n";
     int mode;
     do
     {
         getNumber(prompt,mode, 1, false);
-    } while (mode < 1 || mode > 9 );
+    } while (mode < 1 || mode > 12 );
     switch (mode)
     {
         case 1:
@@ -859,6 +1015,18 @@ int main (void)
             DF->Detach();
             FindStrBufs(DFMgr, selected_ranges);
             break;
+        case 10:
+            DF->Detach();
+            FindData(DFMgr, selected_ranges);
+            break;
+        case 11:
+            DF->Detach();
+            PtrTrace(DFMgr, selected_ranges);
+            break;
+        case 12:
+            DF->Detach();
+            DataPtrTrace(DFMgr, selected_ranges);
+            break;
         default:
             cout << "not implemented :(" << endl;
     }