Search tools run on windows and moved to supported, some windows segment/heap/PE section enumeration work.

New windows dependency: ntdll.lib

library/CMakeLists.txt

@@ -191,7 +191,7 @@ IF(UNIX)
 
   SET(PROJECT_LIBS ${X11_LIBRARY} rt ) #dfhack-md5 dfhack-tixml
 ELSE(UNIX)
-  SET(PROJECT_LIBS psapi)
+  SET(PROJECT_LIBS psapi ${CMAKE_SOURCE_DIR}/library/depends/ntdll/ntdll.lib)
 ENDIF(UNIX)
 
 ADD_LIBRARY(dfhack SHARED ${PROJECT_SRCS})

library/DFProcess-linux-SHM.cpp

@@ -403,6 +403,7 @@ void SHMProcess::getMemRanges( vector<t_memrange> & ranges )
         temp.read = permissions[0] == 'r';
         temp.write = permissions[1] == 'w';
         temp.execute = permissions[2] == 'x';
+        temp.valid = true;
         ranges.push_back(temp);
     }
 }

library/DFProcess-linux-wine.cpp

@@ -219,6 +219,7 @@ void WineProcess::getMemRanges( vector<t_memrange> & ranges )
         temp.read = permissions[0] == 'r';
         temp.write = permissions[1] == 'w';
         temp.execute = permissions[2] == 'x';
+        temp.valid = true;
         ranges.push_back(temp);
     }
 }

library/DFProcess-linux.cpp

@@ -201,6 +201,7 @@ void NormalProcess::getMemRanges( vector<t_memrange> & ranges )
         temp.read = permissions[0] == 'r';
         temp.write = permissions[1] == 'w';
         temp.execute = permissions[2] == 'x';
+        temp.valid = true;
         ranges.push_back(temp);
     }
 }

library/DFProcess-windows-SHM.cpp

@@ -440,6 +440,7 @@ void SHMProcess::getMemRanges( vector<t_memrange> & ranges )
     temp.read = 1;
     temp.write = 1;
     temp.execute = 0; // fake
+    temp.valid = true;
     strcpy(temp.name,"pants");
     ranges.push_back(temp);
 }

library/DFProcess-windows.cpp

@@ -25,6 +25,7 @@ distribution.
 #include "dfhack/DFProcess.h"
 #include "dfhack/VersionInfo.h"
 #include "dfhack/DFError.h"
+#include <string.h>
 using namespace DFHack;
 
 class NormalProcess::Private
@@ -38,6 +39,8 @@ class NormalProcess::Private
             my_pid = 0;
             attached = false;
             suspended = false;
+            base = 0;
+            sections = 0;
         };
         ~Private(){};
         VersionInfo * my_descriptor;
@@ -51,6 +54,9 @@ class NormalProcess::Private
         uint32_t STLSTR_buf_off;
         uint32_t STLSTR_size_off;
         uint32_t STLSTR_cap_off;
+        IMAGE_NT_HEADERS32 pe_header;
+        IMAGE_SECTION_HEADER * sections;
+        uint32_t base;
 };
 
 NormalProcess::NormalProcess(uint32_t pid, vector <VersionInfo *> & known_versions)
@@ -61,8 +67,6 @@ NormalProcess::NormalProcess(uint32_t pid, vector <VersionInfo *> & known_versio
     HANDLE hProcess;
     bool found = false;
 
-    IMAGE_NT_HEADERS32 pe_header;
-    IMAGE_SECTION_HEADER sections[16];
     d->identified = false;
     // open process
     hProcess = OpenProcess( PROCESS_ALL_ACCESS, FALSE, pid );
@@ -78,7 +82,7 @@ NormalProcess::NormalProcess(uint32_t pid, vector <VersionInfo *> & known_versio
     }
 
     // got base ;)
-    uint32_t base = (uint32_t)hmod;
+    d->base = (uint32_t)hmod;
 
     // temporarily assign this to allow some checks
     d->my_handle = hProcess;
@@ -86,9 +90,11 @@ NormalProcess::NormalProcess(uint32_t pid, vector <VersionInfo *> & known_versio
     // read from this process
     try
     {
-        uint32_t pe_offset = readDWord(base+0x3C);
+        uint32_t pe_offset = readDWord(d->base+0x3C);
-        read(base + pe_offset                   , sizeof(pe_header), (uint8_t *)&pe_header);
+        read(d->base + pe_offset                       , sizeof(d->pe_header), (uint8_t *)&d->pe_header);
-        read(base + pe_offset+ sizeof(pe_header), sizeof(sections) , (uint8_t *)&sections );
+        const size_t sectionsSize = sizeof(IMAGE_SECTION_HEADER) * d->pe_header.FileHeader.NumberOfSections;
+        d->sections = (IMAGE_SECTION_HEADER *) malloc(sectionsSize); 
+        read(d->base + pe_offset + sizeof(d->pe_header), sectionsSize, (uint8_t *)d->sections);
         d->my_handle = 0;
     }
     catch (exception &)
@@ -115,7 +121,7 @@ NormalProcess::NormalProcess(uint32_t pid, vector <VersionInfo *> & known_versio
         {
             continue;
         }
-        if (pe_timestamp != pe_header.FileHeader.TimeDateStamp)
+        if (pe_timestamp != d->pe_header.FileHeader.TimeDateStamp)
             continue;
 
         // all went well
@@ -124,7 +130,7 @@ NormalProcess::NormalProcess(uint32_t pid, vector <VersionInfo *> & known_versio
             d->identified = true;
             // give the process a data model and memory layout fixed for the base of first module
             VersionInfo *m = new VersionInfo(**it);
-            m->RebaseAll(base);
+            m->RebaseAll(d->base);
             // keep track of created memory_info object so we can destroy it later
             d->my_descriptor = m;
             m->setParentProcess(this);
@@ -170,6 +176,8 @@ NormalProcess::~NormalProcess()
     {
         CloseHandle(d->my_main_thread);
     }
+    if(d->sections != NULL)
+        free(d->sections);
     delete d;
 }
 
@@ -302,53 +310,111 @@ typedef struct _MEMORY_BASIC_INFORMATION
   uint32_t  Type;
 } MEMORY_BASIC_INFORMATION, *PMEMORY_BASIC_INFORMATION;
 */
+/*
+//Internal structure used to store heap block information.
+struct HeapBlock
+{
+      PVOID dwAddress;
+      DWORD dwSize;
+      DWORD dwFlags;
+      ULONG reserved;
+};
+*/
+void HeapNodes(DWORD pid, map<uint64_t, unsigned int> & heaps)
+{
+    // Create debug buffer
+    PDEBUG_BUFFER db = RtlCreateQueryDebugBuffer(0, FALSE); 
+    // Get process heap data
+    RtlQueryProcessDebugInformation( pid, PDI_HEAPS/* | PDI_HEAP_BLOCKS*/, db);
+    ULONG heapNodeCount = db->HeapInformation ? *PULONG(db->HeapInformation):0;
+    PDEBUG_HEAP_INFORMATION heapInfo = PDEBUG_HEAP_INFORMATION(PULONG(db-> HeapInformation) + 1);
+    // Go through each of the heap nodes and dispaly the information
+    for (unsigned int i = 0; i < heapNodeCount; i++) 
+    {
+        heaps[heapInfo[i].Base] = i;
+    }
+    // Clean up the buffer
+    RtlDestroyQueryDebugBuffer( db );
+}
 
 // FIXME: NEEDS TESTING!
 void NormalProcess::getMemRanges( vector<t_memrange> & ranges )
 {
     MEMORY_BASIC_INFORMATION MBI;
-    DWORD needed;
+    map<uint64_t, unsigned int> heaps;
-    HMODULE hmod;
+    uint64_t movingStart = 0;
-    HMODULE * allModules = 0;
+    map <uint64_t, string> nameMap;
-    bool hasModules = false;
+
     // get page size
     SYSTEM_INFO si;
     GetSystemInfo(&si);
     uint64_t PageSize = si.dwPageSize;
-
+    // enumerate heaps
-    uint64_t page = 0;
+    HeapNodes(d->my_pid, heaps);
-
-    // get all the modules
-    if(EnumProcessModules(this->d->my_handle, &hmod, sizeof(hmod), &needed))
-    {
-        allModules = (HMODULE *) malloc(needed);
-        hasModules = EnumProcessModules(this->d->my_handle, allModules, needed, &needed);
-    }
-
     // go through all the VM regions, convert them to our internal format
-    while (VirtualQueryEx(this->d->my_handle, (const void*) (page * PageSize), &MBI, sizeof(MBI)) == sizeof(MBI))
+    while (VirtualQueryEx(this->d->my_handle, (const void*) (movingStart), &MBI, sizeof(MBI)) == sizeof(MBI))
     {
-        page = MBI.RegionSize / PageSize;
+        movingStart = ((uint64_t)MBI.BaseAddress + MBI.RegionSize);
-        if(MBI.RegionSize - MBI.RegionSize / PageSize != 0)
+        if(movingStart % PageSize != 0)
-            page ++; // skip over non-whole page
+            movingStart = (movingStart / PageSize + 1) * PageSize;
-        if( !(MBI.State & MEM_COMMIT) ) // skip empty regions
+        // skip empty regions and regions we share with other processes (DLLs)
+        if( !(MBI.State & MEM_COMMIT) /*|| !(MBI.Type & MEM_PRIVATE)*/ )
             continue;
-
-        // TODO: we could possibly discard regions shared with other processes (DLLs)?
-        // MBI.Type & MEM_PRIVATE
-
         t_memrange temp;
-        temp.start = (uint64_t) MBI.BaseAddress;
+        temp.start   = (uint64_t) MBI.BaseAddress;
-        temp.end =  ((uint64_t)MBI.BaseAddress + (uint64_t)MBI.RegionSize);
+        temp.end     =  ((uint64_t)MBI.BaseAddress + (uint64_t)MBI.RegionSize);
-        temp.read = MBI.Protect & PAGE_EXECUTE_READ || MBI.Protect & PAGE_EXECUTE_READWRITE || MBI.Protect & PAGE_READONLY || MBI.Protect & PAGE_READWRITE;
+        temp.read    = MBI.Protect & PAGE_EXECUTE_READ || MBI.Protect & PAGE_EXECUTE_READWRITE || MBI.Protect & PAGE_READONLY || MBI.Protect & PAGE_READWRITE;
-        temp.write = MBI.Protect & PAGE_EXECUTE_READWRITE || MBI.Protect & PAGE_READWRITE;
+        temp.write   = MBI.Protect & PAGE_EXECUTE_READWRITE || MBI.Protect & PAGE_READWRITE;
         temp.execute = MBI.Protect & PAGE_EXECUTE_READ || MBI.Protect & PAGE_EXECUTE_READWRITE || MBI.Protect & PAGE_EXECUTE;
-        // FIXME: some relevant description text would be helpful
+        temp.valid = true;
-        strcpy(temp.name,"N/A");
+        if(!GetModuleBaseName(this->d->my_handle, (HMODULE) temp.start, temp.name, 1024))
+        {
+            if(nameMap.count(temp.start))
+            {
+                // potential buffer overflow...
+                strcpy(temp.name, nameMap[temp.start].c_str());
+            }
+            else
+            {
+                // filter away shared segments without a name.
+                if( !(MBI.Type & MEM_PRIVATE) )
+                    continue;
+                else
+                {
+                    // could be a heap?
+                    if(heaps.count(temp.start))
+                    {
+                        sprintf(temp.name,"HEAP %d",heaps[temp.start]);
+                    }
+                    else temp.name[0]=0;
+                }
+                
+
+                
+            }
+        }
+        else
+        {
+            // this is our executable! (could be generalized to pull segments from libs, but whatever)
+            if(d->base == temp.start)
+            {
+                for(int i = 0; i < d->pe_header.FileHeader.NumberOfSections; i++)
+                {
+                    char sectionName[9];
+                    memcpy(sectionName,d->sections[i].Name,8);
+                    sectionName[8] = 0;
+                    string nm;
+                    nm.append(temp.name);
+                    nm.append(" : ");
+                    nm.append(sectionName);
+                    nameMap[temp.start + d->sections[i].VirtualAddress] = nm;
+                }
+            }
+            else
+                continue;
+        }
         ranges.push_back(temp);
     }
-    if(allModules)
-        free(allModules);
 }
 
 uint8_t NormalProcess::readByte (const uint32_t offset)

library/include/dfhack/DFProcess.h

@@ -72,6 +72,7 @@ namespace DFHack
             if (address >= start && address <= end) return true;
             return false;
         }
+        bool valid;
         uint8_t * buffer;
     };
 

library/include/dfhack/modules/Materials.h

@@ -145,7 +145,7 @@ namespace DFHack
 
         void ReadAllMaterials(void);
 
-		std::string getType(t_material & mat);
+        std::string getType(t_material & mat);
         std::string getDescription(t_material & mat);
     private:
         class Private;

library/private/Internal.h

@@ -81,6 +81,57 @@ using namespace std;
     #include <winnt.h>
     #include <psapi.h>
     #include <tlhelp32.h>
+    #include <Dbghelp.h>
+    #pragma comment(lib,"ntdll.lib")
+    #pragma comment(lib,"psapi.lib")
+    typedef LONG NTSTATUS;
+    #define STATUS_SUCCESS ((NTSTATUS)0x00000000L)
+
+    typedef struct _DEBUG_BUFFER {
+    HANDLE SectionHandle;
+    PVOID  SectionBase;
+    PVOID  RemoteSectionBase;
+    ULONG  SectionBaseDelta;
+    HANDLE  EventPairHandle;
+    ULONG  Unknown[2];
+    HANDLE  RemoteThreadHandle;
+    ULONG  InfoClassMask;
+    ULONG  SizeOfInfo;
+    ULONG  AllocatedSize;
+    ULONG  SectionSize;
+    PVOID  ModuleInformation;
+    PVOID  BackTraceInformation;
+    PVOID  HeapInformation;
+    PVOID  LockInformation;
+    PVOID  Reserved[8];
+    } DEBUG_BUFFER, *PDEBUG_BUFFER;
+
+    typedef struct _DEBUG_HEAP_INFORMATION
+    {
+        ULONG Base; // 0×00
+        ULONG Flags; // 0×04
+        USHORT Granularity; // 0×08
+        USHORT Unknown; // 0x0A
+        ULONG Allocated; // 0x0C
+        ULONG Committed; // 0×10
+        ULONG TagCount; // 0×14
+        ULONG BlockCount; // 0×18
+        ULONG Reserved[7]; // 0x1C
+        PVOID Tags; // 0×38
+        PVOID Blocks; // 0x3C
+    } DEBUG_HEAP_INFORMATION, *PDEBUG_HEAP_INFORMATION;
+
+    // RtlQueryProcessDebugInformation.DebugInfoClassMask constants
+    #define PDI_MODULES                       0x01
+    #define PDI_BACKTRACE                     0x02
+    #define PDI_HEAPS                         0x04
+    #define PDI_HEAP_TAGS                     0x08
+    #define PDI_HEAP_BLOCKS                   0x10
+    #define PDI_LOCKS                         0x20
+
+    extern "C" __declspec(dllimport) NTSTATUS __stdcall RtlQueryProcessDebugInformation( IN ULONG  ProcessId, IN ULONG  DebugInfoClassMask, IN OUT PDEBUG_BUFFER  DebugBuffer);
+    extern "C" __declspec(dllimport) PDEBUG_BUFFER __stdcall RtlCreateQueryDebugBuffer( IN ULONG  Size, IN BOOLEAN  EventPair);
+    extern "C" __declspec(dllimport) NTSTATUS __stdcall RtlDestroyQueryDebugBuffer( IN PDEBUG_BUFFER  DebugBuffer);
 #endif
 
 // dfhack dependencies

tools/playground/CMakeLists.txt

@@ -32,15 +32,6 @@ ADD_EXECUTABLE(primitives primitives.cpp)
 #ADD_EXECUTABLE(dfitemdesignator itemdesignator.cpp)
 #TARGET_LINK_LIBRARIES(dfitemdesignator dfhack)
 
-# incrementalsearch - a bit like cheat engine, only DF-specific, very basic
-# and Linux-only
-IF(UNIX)
-    ADD_EXECUTABLE(dfautosearch autosearch.cpp)
-    TARGET_LINK_LIBRARIES(dfautosearch dfhack)
-    ADD_EXECUTABLE(dfincremental incrementalsearch.cpp)
-    TARGET_LINK_LIBRARIES(dfincremental dfhack)
-ENDIF(UNIX)
-
 # catsplosion - Accelerates pregnancy
 # Author: Zhentar
 ADD_EXECUTABLE(dfcatsplosion catsplosion.cpp)
@@ -117,10 +108,3 @@ dfprinttiletypes
 dfhellhole
 RUNTIME DESTINATION bin
 )
-IF(UNIX)
-    install(TARGETS
-    dfautosearch
-    dfincremental
-    RUNTIME DESTINATION bin
-    )
-ENDIF(UNIX)

tools/supported/CMakeLists.txt

@@ -72,6 +72,12 @@ TARGET_LINK_LIBRARIES(dfdoffsets dfhack)
 ADD_EXECUTABLE(dfweather weather.cpp)
 TARGET_LINK_LIBRARIES(dfweather dfhack)
 
+# incrementalsearch - a bit like cheat engine, only DF-specific, very basic
+ADD_EXECUTABLE(dfautosearch autosearch.cpp)
+TARGET_LINK_LIBRARIES(dfautosearch dfhack)
+ADD_EXECUTABLE(dfincremental incrementalsearch.cpp)
+TARGET_LINK_LIBRARIES(dfincremental dfhack)
+
 
 IF(UNIX)
     SET(VEINLOOK_BUILT "NO")
@@ -126,5 +132,7 @@ dfsuspend
 dfflows
 dfliquids
 dfweather
+dfautosearch
+dfincremental
 RUNTIME DESTINATION bin
 )

tools/supported/SegmentedFinder.h

@@ -12,18 +12,37 @@ class SegmentFinder
     {
         _DF = DF;
         mr_ = mr;
-        mr_.buffer = (uint8_t *)malloc (mr_.end - mr_.start);
+        if(mr.valid)
-        DF->ReadRaw(mr_.start,(mr_.end - mr_.start),mr_.buffer);
+        {
-        _SF = SF;
+            mr_.buffer = (uint8_t *)malloc (mr_.end - mr_.start);
+            _SF = SF;
+            try
+            {
+                DF->ReadRaw(mr_.start,(mr_.end - mr_.start),mr_.buffer);
+                valid = true;
+            }
+            catch (DFHack::Error::MemoryAccessDenied &)
+            {
+                free(mr_.buffer);
+                valid = false;
+                mr.valid = false; // mark the range passed in as bad
+                cout << "Range 0x" << hex << mr_.start << " - 0x" <<  mr_.end << dec << " not readable." << endl;
+            }
+        }
     }
     ~SegmentFinder()
     {
-        delete mr_.buffer;
+        if(valid)
+            free(mr_.buffer);
+    }
+    bool isValid()
+    {
+        return valid;
     }
-
     template <class needleType, class hayType, typename comparator >
     bool Find (needleType needle, const uint8_t increment , vector <uint64_t> &newfound, comparator oper)
     {
+        if(!valid) return !newfound.empty();
         //loop
         for(uint64_t offset = 0; offset < (mr_.end - mr_.start) - sizeof(hayType); offset += increment)
         {
@@ -36,6 +55,7 @@ class SegmentFinder
     template < class needleType, class hayType, typename comparator >
     uint64_t FindInRange (needleType needle, comparator oper, uint64_t start, uint64_t length)
     {
+        if(!valid) return 0;
         uint64_t stopper = min((mr_.end - mr_.start) - sizeof(hayType), (start - mr_.start) - sizeof(hayType) + length);
         //loop
         for(uint64_t offset = start - mr_.start; offset < stopper; offset +=1)
@@ -49,6 +69,7 @@ class SegmentFinder
     template <class needleType, class hayType, typename comparator >
     bool Filter (needleType needle, vector <uint64_t> &found, vector <uint64_t> &newfound, comparator oper)
     {
+        if(!valid) return !newfound.empty();
         for( uint64_t i = 0; i < found.size(); i++)
         {
             if(mr_.isInRange(found[i]))
@@ -65,6 +86,7 @@ class SegmentFinder
     SegmentedFinder * _SF;
     DFHack::Context * _DF;
     DFHack::t_memrange mr_;
+    bool valid;
 };
 
 class SegmentedFinder

tools/supported/autosearch.cpp

@@ -391,6 +391,7 @@ bool getRanges(DFHack::Process * p, vector <DFHack::t_memrange>& selected_ranges
         }
         it++;
     }
+    return true;
 }
 
 bool getNumber (string prompt, int & output, int def, bool pdef = true)

tools/supported/incrementalsearch.cpp

@@ -92,6 +92,7 @@ bool getRanges(DFHack::Process * p, vector <DFHack::t_memrange>& selected_ranges
         }
         it++;
     }
+    return true;
 }
 
 bool getNumber (string prompt, int & output, int def, bool pdef = true)