Add an interactive script finding a limited subset of linux offsets.

library/lua/memscan.lua

@@ -0,0 +1,419 @@
+-- Utilities for offset scan scripts.
+
+local _ENV = mkmodule('memscan')
+
+local utils = require('utils')
+
+-- A length-checked view on a memory buffer
+
+CheckedArray = CheckedArray or {}
+
+function CheckedArray.new(type,saddr,eaddr)
+    local data = df.reinterpret_cast(type,saddr)
+    local esize = data:sizeof()
+    local count = math.floor((eaddr-saddr)/esize)
+    local obj = {
+        type = type, start = saddr, size = count*esize,
+        esize = esize, data = data, count = count
+    }
+    setmetatable(obj, CheckedArray)
+    return obj
+end
+
+function CheckedArray:__len()
+    return self.count
+end
+function CheckedArray:__index(idx)
+    if type(idx) == number then
+        if idx >= self.count then
+            error('Index out of bounds: '..tostring(idx))
+        end
+        return self.data[idx]
+    else
+        return CheckedArray[idx]
+    end
+end
+function CheckedArray:__newindex(idx, val)
+    if idx >= self.count then
+        error('Index out of bounds: '..tostring(idx))
+    end
+    self.data[idx] = val
+end
+function CheckedArray:addr2idx(addr, round)
+    local off = addr - self.start
+    if off >= 0 and off < self.size and (round or (off % self.esize) == 0) then
+        return math.floor(off / self.esize), off
+    end
+end
+function CheckedArray:idx2addr(idx)
+    if idx >= 0 and idx < self.count then
+        return self.start + idx*self.esize
+    end
+end
+
+-- Search methods
+
+function CheckedArray:find(data,sidx,eidx,reverse)
+    local dcnt = #data
+    sidx = math.max(0, sidx or 0)
+    eidx = math.min(self.count, eidx or self.count)
+    if (eidx - sidx) >= dcnt and dcnt > 0 then
+        return dfhack.with_temp_object(
+            df.new(self.type, dcnt),
+            function(buffer)
+                for i = 1,dcnt do
+                    buffer[i-1] = data[i]
+                end
+                local cnt = eidx - sidx - dcnt
+                local step = self.esize
+                local sptr = self.start + sidx*step
+                local ksize = dcnt*step
+                if reverse then
+                    local idx, addr = dfhack.internal.memscan(sptr + cnt*step, cnt, -step, buffer, ksize)
+                    if idx then
+                        return sidx + cnt - idx, addr
+                    end
+                else
+                    local idx, addr = dfhack.internal.memscan(sptr, cnt, step, buffer, ksize)
+                    if idx then
+                        return sidx + idx, addr
+                    end
+                end
+            end
+        )
+    end
+end
+function CheckedArray:find_one(data,sidx,eidx,reverse)
+    local idx, addr = self:find(data,sidx,eidx,reverse)
+    if idx then
+        -- Verify this is the only match
+        if reverse then
+            eidx = idx+#data-1
+        else
+            sidx = idx+1
+        end
+        if self:find(data,sidx,eidx,reverse) then
+            return nil
+        end
+    end
+    return idx, addr
+end
+function CheckedArray:list_changes(old_arr,old_val,new_val,delta)
+    if old_arr.type ~= self.type or old_arr.count ~= self.count then
+        error('Incompatible arrays')
+    end
+    local eidx = self.count
+    local optr = old_arr.start
+    local nptr = self.start
+    local esize = self.esize
+    local rv
+    local sidx = 0
+    while true do
+        local idx = dfhack.internal.diffscan(optr, nptr, sidx, eidx, esize, old_val, new_val, delta)
+        if not idx then
+            break
+        end
+        rv = rv or {}
+        rv[#rv+1] = idx
+        sidx = idx+1
+    end
+    return rv
+end
+function CheckedArray:filter_changes(prev_list,old_arr,old_val,new_val,delta)
+    if old_arr.type ~= self.type or old_arr.count ~= self.count then
+        error('Incompatible arrays')
+    end
+    local eidx = self.count
+    local optr = old_arr.start
+    local nptr = self.start
+    local esize = self.esize
+    local rv
+    for i=1,#prev_list do
+        local idx = prev_list[i]
+        if idx < 0 or idx >= eidx then
+            error('Index out of bounds: '..idx)
+        end
+        if dfhack.internal.diffscan(optr, nptr, idx, idx+1, esize, old_val, new_val, delta) then
+            rv = rv or {}
+            rv[#rv+1] = idx
+        end
+    end
+    return rv
+end
+
+-- A raw memory area class
+
+MemoryArea = MemoryArea or {}
+MemoryArea.__index = MemoryArea
+
+function MemoryArea.new(astart, aend)
+    local obj = {
+        start_addr = astart, end_addr = aend, size = aend - astart,
+        int8_t = CheckedArray.new('int8_t',astart,aend),
+        uint8_t = CheckedArray.new('uint8_t',astart,aend),
+        int16_t = CheckedArray.new('int16_t',astart,aend),
+        uint16_t = CheckedArray.new('uint16_t',astart,aend),
+        int32_t = CheckedArray.new('int32_t',astart,aend),
+        uint32_t = CheckedArray.new('uint32_t',astart,aend)
+    }
+    setmetatable(obj, MemoryArea)
+    return obj
+end
+
+function MemoryArea:__gc()
+    df.delete(self.buffer)
+end
+
+function MemoryArea:__tostring()
+    return string.format('<MemoryArea: %x..%x>', self.start_addr, self.end_addr)
+end
+function MemoryArea:contains_range(start,size)
+    return start >= self.start_addr and (start+size) <= self.end_addr
+end
+function MemoryArea:contains_obj(obj,count)
+    local size, base = df.sizeof(obj)
+    return size and base and self:contains_range(base, size*(count or 1))
+end
+
+function MemoryArea:clone()
+    local buffer = df.new('int8_t', self.size)
+    local _, base = buffer:sizeof()
+    local rv = MemoryArea.new(base, base+self.size)
+    rv.buffer = buffer
+    return rv
+end
+function MemoryArea:copy_from(area2)
+    if area2.size ~= self.size then
+        error('Size mismatch')
+    end
+    dfhack.internal.memmove(self.start_addr, area2.start_addr, self.size)
+end
+function MemoryArea:delete()
+    setmetatable(self, nil)
+    df.delete(self.buffer)
+    for k,v in pairs(self) do self[k] = nil end
+end
+
+-- Static data segment search
+
+local function find_data_segment()
+    local data_start, data_end
+
+    for i,mem in ipairs(dfhack.internal.getMemRanges()) do
+        if data_end then
+            if mem.start_addr == data_end and mem.read and mem.write then
+                data_end = mem.end_addr
+            else
+                break
+            end
+        elseif mem.read and mem.write
+           and (string.match(mem.name,'/dwarfort%.exe$')
+             or string.match(mem.name,'/Dwarf_Fortress$'))
+        then
+            data_start = mem.start_addr
+            data_end = mem.end_addr
+        end
+    end
+
+    return data_start, data_end
+end
+
+function get_data_segment()
+    local s, e = find_data_segment()
+    if s and e then
+        return MemoryArea.new(s, e)
+    end
+end
+
+-- Register a found offset, or report an error.
+
+function found_offset(name,val)
+    local cval = dfhack.internal.getAddress(name)
+
+    if not val then
+        print('Could not find offset '..name)
+        if not cval and not utils.prompt_yes_no('Continue with the script?') then
+            error('User quit')
+        end
+        return
+    end
+
+    if df.isvalid(val) then
+        _,val = val:sizeof()
+    end
+
+    print(string.format('Found offset %s: %x', name, val))
+
+    if cval then
+        if cval ~= val then
+            error(string.format('Mismatch with the current value: %x',val))
+        end
+    else
+        dfhack.internal.setAddress(name, val)
+    end
+end
+
+-- Offset of a field within struct
+
+function field_ref(handle,...)
+    local items = table.pack(...)
+    for i=1,items.n-1 do
+        handle = handle[items[i]]
+    end
+    return handle:_field(items[items.n])
+end
+
+function field_offset(type,...)
+    local handle = df.reinterpret_cast(type,1)
+    local _,addr = df.sizeof(field_ref(handle,...))
+    return addr-1
+end
+
+function MemoryArea:object_by_field(addr,type,...)
+    if not addr then
+        return nil
+    end
+    local base = addr - field_offset(type,...)
+    local obj = df.reinterpret_cast(type, base)
+    if not self:contains_obj(obj) then
+        obj = nil
+    end
+    return obj, base
+end
+
+-- Validation
+
+function is_valid_vector(ref,size)
+    local ints = df.reinterpret_cast('uint32_t', ref)
+    return ints[0] <= ints[1] and ints[1] <= ints[2]
+       and (size == nil or (ints[1] - ints[0]) % size == 0)
+end
+
+-- Difference search helper
+
+DiffSearcher = DiffSearcher or {}
+DiffSearcher.__index = DiffSearcher
+
+function DiffSearcher.new(area)
+    local obj = { area = area }
+    setmetatable(obj, DiffSearcher)
+    return obj
+end
+
+function DiffSearcher:begin_search(type)
+    self.type = type
+    self.old_value = nil
+    self.search_sets = nil
+    if not self.save_area then
+        self.save_area = self.area:clone()
+    end
+end
+function DiffSearcher:advance_search(new_value,delta)
+    if self.search_sets then
+        local nsets = #self.search_sets
+        local ovec = self.save_area[self.type]
+        local nvec = self.area[self.type]
+        local new_set
+        if nsets > 0 then
+            local last_set = self.search_sets[nsets]
+            new_set = nvec:filter_changes(last_set,ovec,self.old_value,new_value,delta)
+        else
+            new_set = nvec:list_changes(ovec,self.old_value,new_value,delta)
+        end
+        if new_set then
+            self.search_sets[nsets+1] = new_set
+            self.old_value = new_value
+            self.save_area:copy_from(self.area)
+            return #new_set, new_set
+        end
+    else
+        self.old_value = new_value
+        self.search_sets = {}
+        self.save_area:copy_from(self.area)
+        return #self.save_area[self.type], nil
+    end
+end
+function DiffSearcher:reset()
+    self.search_sets = nil
+    if self.save_area then
+        self.save_area:delete()
+        self.save_area = nil
+    end
+end
+function DiffSearcher:idx2addr(idx)
+    return self.area[self.type]:idx2addr(idx)
+end
+
+-- Menu search utility
+
+function find_menu_cursor(searcher,prompt,data_type,choices,enum)
+    enum = enum or {}
+
+    -- Loop for restarting search from scratch
+    while true do
+        print('\n'..prompt)
+
+        searcher:begin_search(data_type)
+
+        local found = false
+        local ccursor = 0
+
+        -- Loop through choices
+        while true do
+            local choice
+
+            -- Select the next value to search for
+            if type(choices) == 'function' then
+                print('')
+
+                choice = choices(ccursor)
+                ccursor = ccursor + 1
+
+                if not choice then
+                    break
+                end
+            else
+                choice = choices[ccursor+1]
+                ccursor = (ccursor+1) % #choices
+
+                local cname = enum[choice] or choice
+                if type(choice) == 'string' and type(cname) == 'number' then
+                    choice, cname = cname, choice
+                end
+                if cname ~= choice then
+                    cname = cname..' ('..choice..')'
+                end
+
+                -- Ask the user to select it
+                print('\n  Please select: '..cname)
+                if not utils.prompt_yes_no('  Continue?', true) then
+                    break
+                end
+            end
+
+            -- Search for it in the memory
+            local cnt, set = searcher:advance_search(choice)
+            if not cnt then
+                dfhack.printerr('  Converged to zero candidates; probably a mistake somewhere.')
+                break
+            elseif set and cnt == 1 then
+                -- To confirm, wait for two 1-candidate results in a row
+                if found then
+                    local addr = searcher:idx2addr(set[1])
+                    print(string.format('  Confirmed address: %x\n', addr))
+                    return addr, set[1]
+                else
+                    found = true
+                end
+            end
+
+            print('  '..cnt..' candidates remaining.')
+        end
+
+        if not utils.prompt_yes_no('\nRetry search from the start?') then
+            return nil
+        end
+    end
+end
+
+return _ENV

library/lua/utils.lua

@@ -361,4 +361,48 @@ function insert_or_update(vector,item,field,cmp)
     return added,cur,pos
 end
 
+-- Ask a yes-no question
+function prompt_yes_no(msg,default)
+    local prompt = msg
+    if default == nil then
+        prompt = prompt..' (y/n): '
+    elseif default then
+        prompt = prompt..' (y/n/enter=y): '
+    else
+        prompt = prompt..' (y/n/enter=n): '
+    end
+    while true do
+        local rv = dfhack.lineedit(prompt)
+        if rv then
+            if string.match(rv,'^[Yy]') then
+                return true
+            elseif string.match(rv,'^[Nn]') then
+                return false
+            elseif rv == '' and default ~= nil then
+                return default
+            end
+        end
+    end
+end
+
+-- Ask for input with check function
+function prompt_input(prompt,check,quit_str)
+    quit_str = quit_str or '~~~'
+    while true do
+        local rv = dfhack.lineedit(prompt)
+        if rv == quit_str then
+            return nil
+        end
+        local rtbl = table.pack(check(rv))
+        if rtbl[1] then
+            return table.unpack(rtbl,2,rtbl.n)
+        end
+    end
+end
+
+function check_number(text)
+    local nv = tonumber(text)
+    return nv ~= nil, nv
+end
+
 return _ENV

scripts/devel/find-offsets.lua

@@ -0,0 +1,368 @@
+-- Find some offsets for linux.
+
+local utils = require 'utils'
+local ms = require 'memscan'
+
+local scan_all = false
+local is_known = dfhack.internal.getAddress
+
+collectgarbage()
+
+print[[
+WARNING: THIS SCRIPT IS STRICTLY FOR DFHACK DEVELOPERS.
+
+Running this script on a new DF version will NOT
+MAKE IT RUN CORRECTLY if any data structures
+changed, thus possibly leading to CRASHES AND/OR
+PERMANENT SAVE CORRUPTION.
+
+This script should be initially started immediately
+after loading the game, WITHOUT first loading a world.
+It expects vanilla game configuration, without any
+custom tilesets or init file changes.
+]]
+
+if not utils.prompt_yes_no('Proceed?') then
+    return
+end
+
+-- Data segment location
+
+local data = ms.get_data_segment()
+if not data then
+    error('Could not find data segment')
+end
+
+print('Data section: '..tostring(data))
+if data.size < 5000000 then
+    error('Data segment too short.')
+end
+
+local searcher = ms.DiffSearcher.new(data)
+
+local function validate_offset(name,validator,addr,tname,...)
+    local obj = data:object_by_field(addr,tname,...)
+    if obj and not validator(obj) then
+        obj = nil
+    end
+    ms.found_offset(name,obj)
+end
+
+--
+-- Cursor group
+--
+
+local function find_cursor()
+    print('\nPlease navigate to the title screen to find cursor.')
+    if not utils.prompt_yes_no('Proceed?', true) then
+        return false
+    end
+
+    -- Unpadded version
+    local idx, addr = data.int32_t:find_one{
+        -30000, -30000, -30000,
+        -30000, -30000, -30000, -30000, -30000, -30000,
+        df.game_mode.NONE, df.game_type.NONE
+    }
+    if idx then
+        ms.found_offset('cursor', addr)
+        ms.found_offset('selection_rect', addr + 12)
+        ms.found_offset('gamemode', addr + 12 + 24)
+        ms.found_offset('gametype', addr + 12 + 24 + 4)
+        return true
+    end
+
+    -- Padded version
+    idx, addr = data.int32_t:find_one{
+        -30000, -30000, -30000, 0,
+        -30000, -30000, -30000, -30000, -30000, -30000, 0, 0,
+        df.game_mode.NONE, 0, 0, 0, df.game_type.NONE
+    }
+    if idx then
+        ms.found_offset('cursor', addr)
+        ms.found_offset('selection_rect', addr + 0x10)
+        ms.found_offset('gamemode', addr + 0x30)
+        ms.found_offset('gametype', addr + 0x40)
+        return true
+    end
+
+    dfhack.printerr('Could not find cursor.')
+    return false
+end
+
+if scan_all or not (
+    is_known 'cursor' and is_known 'selection_rect' and
+    is_known 'gamemode' and is_known 'gametype'
+) then
+    find_cursor()
+end
+
+--
+-- Announcements
+--
+
+local function find_announcements()
+    idx, addr = data.int32_t:find_one{
+        25, 25, 31, 31, 24, 24, 40, 40, 40, 40, 40, 40, 40
+    }
+    if idx then
+        ms.found_offset('announcements', addr)
+        return
+    end
+
+    dfhack.printerr('Could not find announcements.')
+end
+
+if scan_all or not is_known 'announcements' then
+    find_announcements()
+end
+
+--
+-- d_init
+--
+
+local function is_valid_d_init(di)
+    if di.sky_tile ~= 178 then
+        print('Sky tile expected 178, found: '..di.sky_tile)
+        if not utils.prompt_yes_no('Ignore?') then
+            return false
+        end
+    end
+
+    local ann = is_known 'announcements'
+    local size,ptr = di:sizeof()
+    if ann and ptr+size ~= ann then
+        print('Announcements not immediately after d_init.')
+        if not utils.prompt_yes_no('Ignore?') then
+            return false
+        end
+    end
+
+    return true
+end
+
+local function find_d_init()
+    idx, addr = data.int16_t:find_one{
+        1,0, 2,0, 5,0, 25,0,   -- path_cost
+        4,4,                   -- embark_rect
+        20,1000,1000,1000,1000 -- store_dist
+    }
+    if idx then
+        validate_offset('d_init', is_valid_d_init, addr, df.d_init, 'path_cost')
+        return
+    end
+
+    dfhack.printerr('Could not find d_init')
+end
+
+if scan_all or not is_known 'd_init' then
+    find_d_init()
+end
+
+--
+-- World
+--
+
+local function is_valid_world(world)
+    if not ms.is_valid_vector(world.units.all, 4)
+    or not ms.is_valid_vector(world.units.bad, 4)
+    or not ms.is_valid_vector(world.history.figures, 4)
+    or not ms.is_valid_vector(world.cur_savegame.map_features, 4)
+    then
+        dfhack.printerr('Vector layout check failed.')
+        return false
+    end
+
+    if #world.units.all == 0 or #world.units.all ~= #world.units.bad then
+        print('Different or zero size of units.all and units.bad:'..#world.units.all..' vs '..#world.units.bad)
+        if not utils.prompt_yes_no('Ignore?') then
+            return false
+        end
+    end
+
+    return true
+end
+
+local function find_world()
+    local addr = ms.find_menu_cursor(
+        searcher, [[
+Searching for world. Please open the stockpile creation
+menu, and follow instructions below:]],
+        'int32_t',
+        { 'Corpses', 'Refuse', 'Stone', 'Wood', 'Gems', 'Bars', 'Cloth', 'Leather', 'Ammo', 'Coins' },
+        df.stockpile_category
+    )
+    validate_offset('world', is_valid_world, addr, df.world, 'selected_stockpile_type')
+end
+
+if scan_all or not is_known 'world' then
+    find_world()
+end
+
+--
+-- UI
+--
+
+local function is_valid_ui(ui)
+    if not ms.is_valid_vector(ui.economic_stone, 1)
+    or not ms.is_valid_vector(ui.dipscripts, 4)
+    then
+        dfhack.printerr('Vector layout check failed.')
+        return false
+    end
+
+    if ui.follow_item ~= -1 or ui.follow_unit ~= -1 then
+        print('Invalid follow state: '..ui.follow_item..', '..ui.follow_unit)
+        return false
+    end
+
+    return true
+end
+
+local function find_ui()
+    local addr = ms.find_menu_cursor(
+        searcher, [[
+Searching for ui. Please open the designation
+menu, and follow instructions below:]],
+        'int16_t',
+        { 'DesignateMine', 'DesignateChannel', 'DesignateRemoveRamps', 'DesignateUpStair',
+          'DesignateDownStair', 'DesignateUpDownStair', 'DesignateUpRamp', 'DesignateChopTrees' },
+        df.ui_sidebar_mode
+    )
+    validate_offset('ui', is_valid_ui, addr, df.ui, 'main', 'mode')
+end
+
+if scan_all or not is_known 'ui' then
+    find_ui()
+end
+
+--
+-- ui_sidebar_menus
+--
+
+local function is_valid_ui_sidebar_menus(usm)
+    if not ms.is_valid_vector(usm.workshop_job.choices_all, 4)
+    or not ms.is_valid_vector(usm.workshop_job.choices_visible, 4)
+    then
+        dfhack.printerr('Vector layout check failed.')
+        return false
+    end
+
+    if #usm.workshop_job.choices_all == 0
+    or #usm.workshop_job.choices_all ~= #usm.workshop_job.choices_visible then
+        print('Different or zero size of visible and all choices:'..
+              #usm.workshop_job.choices_all..' vs '..#usm.workshop_job.choices_visible)
+        if not utils.prompt_yes_no('Ignore?') then
+            return false
+        end
+    end
+
+    return true
+end
+
+local function find_ui_sidebar_menus()
+    local addr = ms.find_menu_cursor(
+        searcher, [[
+Searching for ui_sidebar_menus. Please open the add job
+ui of Mason, Craftsdwarfs, or Carpenters workshop, and
+select entries in the list:]],
+        'int32_t',
+        { 0, 1, 2, 3, 4, 5, 6 }
+    )
+    validate_offset('ui_sidebar_menus', is_valid_ui_sidebar_menus,
+                    addr, df.ui_sidebar_menus, 'workshop_job', 'cursor')
+end
+
+if scan_all or not is_known 'ui_sidebar_menus' then
+    find_ui_sidebar_menus()
+end
+
+--
+-- ui_build_selector
+--
+
+local function is_valid_ui_build_selector(ubs)
+    if not ms.is_valid_vector(ubs.requirements, 4)
+    or not ms.is_valid_vector(ubs.choices, 4)
+    then
+        dfhack.printerr('Vector layout check failed.')
+        return false
+    end
+
+    if ubs.building_type ~= df.building_type.Trap
+    or ubs.building_subtype ~= df.trap_type.PressurePlate then
+        print('Invalid building type and subtype:'..ubs.building_type..','..ubs.building_subtype)
+        return false
+    end
+
+    return true
+end
+
+local function find_ui_build_selector()
+    local addr = ms.find_menu_cursor(
+        searcher, [[
+Searching for ui_build_selector. Please start constructing
+a pressure plate, and enable creatures. Then change the min
+weight as requested, remembering that the ui truncates the
+number, so when it shows "Min (5000df", it means 50000:]],
+        'int32_t',
+        { 50000, 49000, 48000, 47000, 46000, 45000, 44000 }
+    )
+    validate_offset('ui_build_selector', is_valid_ui_build_selector,
+                    addr, df.ui_build_selector, 'plate_info', 'unit_min')
+end
+
+if scan_all or not is_known 'ui_build_selector' then
+    find_ui_build_selector()
+end
+
+--
+-- ui_selected_unit
+--
+
+local function find_ui_selected_unit()
+    if not is_known 'world' then
+        dfhack.printerr('Cannot find ui_selected_unit: no world')
+        return
+    end
+
+    for i,unit in ipairs(df.global.world.units.active) do
+        dfhack.units.setNickname(unit, i)
+    end
+
+    local addr = ms.find_menu_cursor(
+        searcher, [[
+Searching for ui_selected_unit. Please activate the 'v'
+mode, point it at units, and enter their numeric nickname
+into the prompts below:]],
+        'int32_t',
+        function()
+            return utils.prompt_input('  Enter index: ', utils.check_number)
+        end
+    )
+    ms.found_offset('ui_selected_unit', addr)
+end
+
+if scan_all or not is_known 'ui_selected_unit' then
+    find_ui_selected_unit()
+end
+
+--
+-- ui_unit_view_mode
+--
+
+local function find_ui_unit_view_mode()
+    local addr = ms.find_menu_cursor(
+        searcher, [[
+Searching for ui_unit_view_mode. Having selected a unit
+with 'v', switch the pages as requested:]],
+        'int32_t',
+        { 'General', 'Inventory', 'Preferences', 'Wounds' },
+        df.ui_unit_view_mode.T_value
+    )
+    ms.found_offset('ui_unit_view_mode', addr)
+end
+
+if scan_all or not is_known 'ui_unit_view_mode' then
+    find_ui_unit_view_mode()
+end