From 67536da2fe718b103f4200b95f00cd30000cb7ad Mon Sep 17 00:00:00 2001 From: Alexander Gavrilov Date: Sat, 16 Jun 2012 19:51:15 +0400 Subject: [PATCH] Add an interactive script finding a limited subset of linux offsets. --- library/lua/memscan.lua | 419 +++++++++++++++++++++++++++++++++ library/lua/utils.lua | 44 ++++ scripts/devel/find-offsets.lua | 368 +++++++++++++++++++++++++++++ 3 files changed, 831 insertions(+) create mode 100644 library/lua/memscan.lua create mode 100644 scripts/devel/find-offsets.lua diff --git a/library/lua/memscan.lua b/library/lua/memscan.lua new file mode 100644 index 000000000..e5a0c6f72 --- /dev/null +++ b/library/lua/memscan.lua @@ -0,0 +1,419 @@ +-- Utilities for offset scan scripts. + +local _ENV = mkmodule('memscan') + +local utils = require('utils') + +-- A length-checked view on a memory buffer + +CheckedArray = CheckedArray or {} + +function CheckedArray.new(type,saddr,eaddr) + local data = df.reinterpret_cast(type,saddr) + local esize = data:sizeof() + local count = math.floor((eaddr-saddr)/esize) + local obj = { + type = type, start = saddr, size = count*esize, + esize = esize, data = data, count = count + } + setmetatable(obj, CheckedArray) + return obj +end + +function CheckedArray:__len() + return self.count +end +function CheckedArray:__index(idx) + if type(idx) == number then + if idx >= self.count then + error('Index out of bounds: '..tostring(idx)) + end + return self.data[idx] + else + return CheckedArray[idx] + end +end +function CheckedArray:__newindex(idx, val) + if idx >= self.count then + error('Index out of bounds: '..tostring(idx)) + end + self.data[idx] = val +end +function CheckedArray:addr2idx(addr, round) + local off = addr - self.start + if off >= 0 and off < self.size and (round or (off % self.esize) == 0) then + return math.floor(off / self.esize), off + end +end +function CheckedArray:idx2addr(idx) + if idx >= 0 and idx < self.count then + return self.start + idx*self.esize + end +end + +-- Search methods + +function CheckedArray:find(data,sidx,eidx,reverse) + local dcnt = #data + sidx = math.max(0, sidx or 0) + eidx = math.min(self.count, eidx or self.count) + if (eidx - sidx) >= dcnt and dcnt > 0 then + return dfhack.with_temp_object( + df.new(self.type, dcnt), + function(buffer) + for i = 1,dcnt do + buffer[i-1] = data[i] + end + local cnt = eidx - sidx - dcnt + local step = self.esize + local sptr = self.start + sidx*step + local ksize = dcnt*step + if reverse then + local idx, addr = dfhack.internal.memscan(sptr + cnt*step, cnt, -step, buffer, ksize) + if idx then + return sidx + cnt - idx, addr + end + else + local idx, addr = dfhack.internal.memscan(sptr, cnt, step, buffer, ksize) + if idx then + return sidx + idx, addr + end + end + end + ) + end +end +function CheckedArray:find_one(data,sidx,eidx,reverse) + local idx, addr = self:find(data,sidx,eidx,reverse) + if idx then + -- Verify this is the only match + if reverse then + eidx = idx+#data-1 + else + sidx = idx+1 + end + if self:find(data,sidx,eidx,reverse) then + return nil + end + end + return idx, addr +end +function CheckedArray:list_changes(old_arr,old_val,new_val,delta) + if old_arr.type ~= self.type or old_arr.count ~= self.count then + error('Incompatible arrays') + end + local eidx = self.count + local optr = old_arr.start + local nptr = self.start + local esize = self.esize + local rv + local sidx = 0 + while true do + local idx = dfhack.internal.diffscan(optr, nptr, sidx, eidx, esize, old_val, new_val, delta) + if not idx then + break + end + rv = rv or {} + rv[#rv+1] = idx + sidx = idx+1 + end + return rv +end +function CheckedArray:filter_changes(prev_list,old_arr,old_val,new_val,delta) + if old_arr.type ~= self.type or old_arr.count ~= self.count then + error('Incompatible arrays') + end + local eidx = self.count + local optr = old_arr.start + local nptr = self.start + local esize = self.esize + local rv + for i=1,#prev_list do + local idx = prev_list[i] + if idx < 0 or idx >= eidx then + error('Index out of bounds: '..idx) + end + if dfhack.internal.diffscan(optr, nptr, idx, idx+1, esize, old_val, new_val, delta) then + rv = rv or {} + rv[#rv+1] = idx + end + end + return rv +end + +-- A raw memory area class + +MemoryArea = MemoryArea or {} +MemoryArea.__index = MemoryArea + +function MemoryArea.new(astart, aend) + local obj = { + start_addr = astart, end_addr = aend, size = aend - astart, + int8_t = CheckedArray.new('int8_t',astart,aend), + uint8_t = CheckedArray.new('uint8_t',astart,aend), + int16_t = CheckedArray.new('int16_t',astart,aend), + uint16_t = CheckedArray.new('uint16_t',astart,aend), + int32_t = CheckedArray.new('int32_t',astart,aend), + uint32_t = CheckedArray.new('uint32_t',astart,aend) + } + setmetatable(obj, MemoryArea) + return obj +end + +function MemoryArea:__gc() + df.delete(self.buffer) +end + +function MemoryArea:__tostring() + return string.format('', self.start_addr, self.end_addr) +end +function MemoryArea:contains_range(start,size) + return start >= self.start_addr and (start+size) <= self.end_addr +end +function MemoryArea:contains_obj(obj,count) + local size, base = df.sizeof(obj) + return size and base and self:contains_range(base, size*(count or 1)) +end + +function MemoryArea:clone() + local buffer = df.new('int8_t', self.size) + local _, base = buffer:sizeof() + local rv = MemoryArea.new(base, base+self.size) + rv.buffer = buffer + return rv +end +function MemoryArea:copy_from(area2) + if area2.size ~= self.size then + error('Size mismatch') + end + dfhack.internal.memmove(self.start_addr, area2.start_addr, self.size) +end +function MemoryArea:delete() + setmetatable(self, nil) + df.delete(self.buffer) + for k,v in pairs(self) do self[k] = nil end +end + +-- Static data segment search + +local function find_data_segment() + local data_start, data_end + + for i,mem in ipairs(dfhack.internal.getMemRanges()) do + if data_end then + if mem.start_addr == data_end and mem.read and mem.write then + data_end = mem.end_addr + else + break + end + elseif mem.read and mem.write + and (string.match(mem.name,'/dwarfort%.exe$') + or string.match(mem.name,'/Dwarf_Fortress$')) + then + data_start = mem.start_addr + data_end = mem.end_addr + end + end + + return data_start, data_end +end + +function get_data_segment() + local s, e = find_data_segment() + if s and e then + return MemoryArea.new(s, e) + end +end + +-- Register a found offset, or report an error. + +function found_offset(name,val) + local cval = dfhack.internal.getAddress(name) + + if not val then + print('Could not find offset '..name) + if not cval and not utils.prompt_yes_no('Continue with the script?') then + error('User quit') + end + return + end + + if df.isvalid(val) then + _,val = val:sizeof() + end + + print(string.format('Found offset %s: %x', name, val)) + + if cval then + if cval ~= val then + error(string.format('Mismatch with the current value: %x',val)) + end + else + dfhack.internal.setAddress(name, val) + end +end + +-- Offset of a field within struct + +function field_ref(handle,...) + local items = table.pack(...) + for i=1,items.n-1 do + handle = handle[items[i]] + end + return handle:_field(items[items.n]) +end + +function field_offset(type,...) + local handle = df.reinterpret_cast(type,1) + local _,addr = df.sizeof(field_ref(handle,...)) + return addr-1 +end + +function MemoryArea:object_by_field(addr,type,...) + if not addr then + return nil + end + local base = addr - field_offset(type,...) + local obj = df.reinterpret_cast(type, base) + if not self:contains_obj(obj) then + obj = nil + end + return obj, base +end + +-- Validation + +function is_valid_vector(ref,size) + local ints = df.reinterpret_cast('uint32_t', ref) + return ints[0] <= ints[1] and ints[1] <= ints[2] + and (size == nil or (ints[1] - ints[0]) % size == 0) +end + +-- Difference search helper + +DiffSearcher = DiffSearcher or {} +DiffSearcher.__index = DiffSearcher + +function DiffSearcher.new(area) + local obj = { area = area } + setmetatable(obj, DiffSearcher) + return obj +end + +function DiffSearcher:begin_search(type) + self.type = type + self.old_value = nil + self.search_sets = nil + if not self.save_area then + self.save_area = self.area:clone() + end +end +function DiffSearcher:advance_search(new_value,delta) + if self.search_sets then + local nsets = #self.search_sets + local ovec = self.save_area[self.type] + local nvec = self.area[self.type] + local new_set + if nsets > 0 then + local last_set = self.search_sets[nsets] + new_set = nvec:filter_changes(last_set,ovec,self.old_value,new_value,delta) + else + new_set = nvec:list_changes(ovec,self.old_value,new_value,delta) + end + if new_set then + self.search_sets[nsets+1] = new_set + self.old_value = new_value + self.save_area:copy_from(self.area) + return #new_set, new_set + end + else + self.old_value = new_value + self.search_sets = {} + self.save_area:copy_from(self.area) + return #self.save_area[self.type], nil + end +end +function DiffSearcher:reset() + self.search_sets = nil + if self.save_area then + self.save_area:delete() + self.save_area = nil + end +end +function DiffSearcher:idx2addr(idx) + return self.area[self.type]:idx2addr(idx) +end + +-- Menu search utility + +function find_menu_cursor(searcher,prompt,data_type,choices,enum) + enum = enum or {} + + -- Loop for restarting search from scratch + while true do + print('\n'..prompt) + + searcher:begin_search(data_type) + + local found = false + local ccursor = 0 + + -- Loop through choices + while true do + local choice + + -- Select the next value to search for + if type(choices) == 'function' then + print('') + + choice = choices(ccursor) + ccursor = ccursor + 1 + + if not choice then + break + end + else + choice = choices[ccursor+1] + ccursor = (ccursor+1) % #choices + + local cname = enum[choice] or choice + if type(choice) == 'string' and type(cname) == 'number' then + choice, cname = cname, choice + end + if cname ~= choice then + cname = cname..' ('..choice..')' + end + + -- Ask the user to select it + print('\n Please select: '..cname) + if not utils.prompt_yes_no(' Continue?', true) then + break + end + end + + -- Search for it in the memory + local cnt, set = searcher:advance_search(choice) + if not cnt then + dfhack.printerr(' Converged to zero candidates; probably a mistake somewhere.') + break + elseif set and cnt == 1 then + -- To confirm, wait for two 1-candidate results in a row + if found then + local addr = searcher:idx2addr(set[1]) + print(string.format(' Confirmed address: %x\n', addr)) + return addr, set[1] + else + found = true + end + end + + print(' '..cnt..' candidates remaining.') + end + + if not utils.prompt_yes_no('\nRetry search from the start?') then + return nil + end + end +end + +return _ENV diff --git a/library/lua/utils.lua b/library/lua/utils.lua index e67801f4f..4fac56ece 100644 --- a/library/lua/utils.lua +++ b/library/lua/utils.lua @@ -361,4 +361,48 @@ function insert_or_update(vector,item,field,cmp) return added,cur,pos end +-- Ask a yes-no question +function prompt_yes_no(msg,default) + local prompt = msg + if default == nil then + prompt = prompt..' (y/n): ' + elseif default then + prompt = prompt..' (y/n/enter=y): ' + else + prompt = prompt..' (y/n/enter=n): ' + end + while true do + local rv = dfhack.lineedit(prompt) + if rv then + if string.match(rv,'^[Yy]') then + return true + elseif string.match(rv,'^[Nn]') then + return false + elseif rv == '' and default ~= nil then + return default + end + end + end +end + +-- Ask for input with check function +function prompt_input(prompt,check,quit_str) + quit_str = quit_str or '~~~' + while true do + local rv = dfhack.lineedit(prompt) + if rv == quit_str then + return nil + end + local rtbl = table.pack(check(rv)) + if rtbl[1] then + return table.unpack(rtbl,2,rtbl.n) + end + end +end + +function check_number(text) + local nv = tonumber(text) + return nv ~= nil, nv +end + return _ENV \ No newline at end of file diff --git a/scripts/devel/find-offsets.lua b/scripts/devel/find-offsets.lua new file mode 100644 index 000000000..ef16026e3 --- /dev/null +++ b/scripts/devel/find-offsets.lua @@ -0,0 +1,368 @@ +-- Find some offsets for linux. + +local utils = require 'utils' +local ms = require 'memscan' + +local scan_all = false +local is_known = dfhack.internal.getAddress + +collectgarbage() + +print[[ +WARNING: THIS SCRIPT IS STRICTLY FOR DFHACK DEVELOPERS. + +Running this script on a new DF version will NOT +MAKE IT RUN CORRECTLY if any data structures +changed, thus possibly leading to CRASHES AND/OR +PERMANENT SAVE CORRUPTION. + +This script should be initially started immediately +after loading the game, WITHOUT first loading a world. +It expects vanilla game configuration, without any +custom tilesets or init file changes. +]] + +if not utils.prompt_yes_no('Proceed?') then + return +end + +-- Data segment location + +local data = ms.get_data_segment() +if not data then + error('Could not find data segment') +end + +print('Data section: '..tostring(data)) +if data.size < 5000000 then + error('Data segment too short.') +end + +local searcher = ms.DiffSearcher.new(data) + +local function validate_offset(name,validator,addr,tname,...) + local obj = data:object_by_field(addr,tname,...) + if obj and not validator(obj) then + obj = nil + end + ms.found_offset(name,obj) +end + +-- +-- Cursor group +-- + +local function find_cursor() + print('\nPlease navigate to the title screen to find cursor.') + if not utils.prompt_yes_no('Proceed?', true) then + return false + end + + -- Unpadded version + local idx, addr = data.int32_t:find_one{ + -30000, -30000, -30000, + -30000, -30000, -30000, -30000, -30000, -30000, + df.game_mode.NONE, df.game_type.NONE + } + if idx then + ms.found_offset('cursor', addr) + ms.found_offset('selection_rect', addr + 12) + ms.found_offset('gamemode', addr + 12 + 24) + ms.found_offset('gametype', addr + 12 + 24 + 4) + return true + end + + -- Padded version + idx, addr = data.int32_t:find_one{ + -30000, -30000, -30000, 0, + -30000, -30000, -30000, -30000, -30000, -30000, 0, 0, + df.game_mode.NONE, 0, 0, 0, df.game_type.NONE + } + if idx then + ms.found_offset('cursor', addr) + ms.found_offset('selection_rect', addr + 0x10) + ms.found_offset('gamemode', addr + 0x30) + ms.found_offset('gametype', addr + 0x40) + return true + end + + dfhack.printerr('Could not find cursor.') + return false +end + +if scan_all or not ( + is_known 'cursor' and is_known 'selection_rect' and + is_known 'gamemode' and is_known 'gametype' +) then + find_cursor() +end + +-- +-- Announcements +-- + +local function find_announcements() + idx, addr = data.int32_t:find_one{ + 25, 25, 31, 31, 24, 24, 40, 40, 40, 40, 40, 40, 40 + } + if idx then + ms.found_offset('announcements', addr) + return + end + + dfhack.printerr('Could not find announcements.') +end + +if scan_all or not is_known 'announcements' then + find_announcements() +end + +-- +-- d_init +-- + +local function is_valid_d_init(di) + if di.sky_tile ~= 178 then + print('Sky tile expected 178, found: '..di.sky_tile) + if not utils.prompt_yes_no('Ignore?') then + return false + end + end + + local ann = is_known 'announcements' + local size,ptr = di:sizeof() + if ann and ptr+size ~= ann then + print('Announcements not immediately after d_init.') + if not utils.prompt_yes_no('Ignore?') then + return false + end + end + + return true +end + +local function find_d_init() + idx, addr = data.int16_t:find_one{ + 1,0, 2,0, 5,0, 25,0, -- path_cost + 4,4, -- embark_rect + 20,1000,1000,1000,1000 -- store_dist + } + if idx then + validate_offset('d_init', is_valid_d_init, addr, df.d_init, 'path_cost') + return + end + + dfhack.printerr('Could not find d_init') +end + +if scan_all or not is_known 'd_init' then + find_d_init() +end + +-- +-- World +-- + +local function is_valid_world(world) + if not ms.is_valid_vector(world.units.all, 4) + or not ms.is_valid_vector(world.units.bad, 4) + or not ms.is_valid_vector(world.history.figures, 4) + or not ms.is_valid_vector(world.cur_savegame.map_features, 4) + then + dfhack.printerr('Vector layout check failed.') + return false + end + + if #world.units.all == 0 or #world.units.all ~= #world.units.bad then + print('Different or zero size of units.all and units.bad:'..#world.units.all..' vs '..#world.units.bad) + if not utils.prompt_yes_no('Ignore?') then + return false + end + end + + return true +end + +local function find_world() + local addr = ms.find_menu_cursor( + searcher, [[ +Searching for world. Please open the stockpile creation +menu, and follow instructions below:]], + 'int32_t', + { 'Corpses', 'Refuse', 'Stone', 'Wood', 'Gems', 'Bars', 'Cloth', 'Leather', 'Ammo', 'Coins' }, + df.stockpile_category + ) + validate_offset('world', is_valid_world, addr, df.world, 'selected_stockpile_type') +end + +if scan_all or not is_known 'world' then + find_world() +end + +-- +-- UI +-- + +local function is_valid_ui(ui) + if not ms.is_valid_vector(ui.economic_stone, 1) + or not ms.is_valid_vector(ui.dipscripts, 4) + then + dfhack.printerr('Vector layout check failed.') + return false + end + + if ui.follow_item ~= -1 or ui.follow_unit ~= -1 then + print('Invalid follow state: '..ui.follow_item..', '..ui.follow_unit) + return false + end + + return true +end + +local function find_ui() + local addr = ms.find_menu_cursor( + searcher, [[ +Searching for ui. Please open the designation +menu, and follow instructions below:]], + 'int16_t', + { 'DesignateMine', 'DesignateChannel', 'DesignateRemoveRamps', 'DesignateUpStair', + 'DesignateDownStair', 'DesignateUpDownStair', 'DesignateUpRamp', 'DesignateChopTrees' }, + df.ui_sidebar_mode + ) + validate_offset('ui', is_valid_ui, addr, df.ui, 'main', 'mode') +end + +if scan_all or not is_known 'ui' then + find_ui() +end + +-- +-- ui_sidebar_menus +-- + +local function is_valid_ui_sidebar_menus(usm) + if not ms.is_valid_vector(usm.workshop_job.choices_all, 4) + or not ms.is_valid_vector(usm.workshop_job.choices_visible, 4) + then + dfhack.printerr('Vector layout check failed.') + return false + end + + if #usm.workshop_job.choices_all == 0 + or #usm.workshop_job.choices_all ~= #usm.workshop_job.choices_visible then + print('Different or zero size of visible and all choices:'.. + #usm.workshop_job.choices_all..' vs '..#usm.workshop_job.choices_visible) + if not utils.prompt_yes_no('Ignore?') then + return false + end + end + + return true +end + +local function find_ui_sidebar_menus() + local addr = ms.find_menu_cursor( + searcher, [[ +Searching for ui_sidebar_menus. Please open the add job +ui of Mason, Craftsdwarfs, or Carpenters workshop, and +select entries in the list:]], + 'int32_t', + { 0, 1, 2, 3, 4, 5, 6 } + ) + validate_offset('ui_sidebar_menus', is_valid_ui_sidebar_menus, + addr, df.ui_sidebar_menus, 'workshop_job', 'cursor') +end + +if scan_all or not is_known 'ui_sidebar_menus' then + find_ui_sidebar_menus() +end + +-- +-- ui_build_selector +-- + +local function is_valid_ui_build_selector(ubs) + if not ms.is_valid_vector(ubs.requirements, 4) + or not ms.is_valid_vector(ubs.choices, 4) + then + dfhack.printerr('Vector layout check failed.') + return false + end + + if ubs.building_type ~= df.building_type.Trap + or ubs.building_subtype ~= df.trap_type.PressurePlate then + print('Invalid building type and subtype:'..ubs.building_type..','..ubs.building_subtype) + return false + end + + return true +end + +local function find_ui_build_selector() + local addr = ms.find_menu_cursor( + searcher, [[ +Searching for ui_build_selector. Please start constructing +a pressure plate, and enable creatures. Then change the min +weight as requested, remembering that the ui truncates the +number, so when it shows "Min (5000df", it means 50000:]], + 'int32_t', + { 50000, 49000, 48000, 47000, 46000, 45000, 44000 } + ) + validate_offset('ui_build_selector', is_valid_ui_build_selector, + addr, df.ui_build_selector, 'plate_info', 'unit_min') +end + +if scan_all or not is_known 'ui_build_selector' then + find_ui_build_selector() +end + +-- +-- ui_selected_unit +-- + +local function find_ui_selected_unit() + if not is_known 'world' then + dfhack.printerr('Cannot find ui_selected_unit: no world') + return + end + + for i,unit in ipairs(df.global.world.units.active) do + dfhack.units.setNickname(unit, i) + end + + local addr = ms.find_menu_cursor( + searcher, [[ +Searching for ui_selected_unit. Please activate the 'v' +mode, point it at units, and enter their numeric nickname +into the prompts below:]], + 'int32_t', + function() + return utils.prompt_input(' Enter index: ', utils.check_number) + end + ) + ms.found_offset('ui_selected_unit', addr) +end + +if scan_all or not is_known 'ui_selected_unit' then + find_ui_selected_unit() +end + +-- +-- ui_unit_view_mode +-- + +local function find_ui_unit_view_mode() + local addr = ms.find_menu_cursor( + searcher, [[ +Searching for ui_unit_view_mode. Having selected a unit +with 'v', switch the pages as requested:]], + 'int32_t', + { 'General', 'Inventory', 'Preferences', 'Wounds' }, + df.ui_unit_view_mode.T_value + ) + ms.found_offset('ui_unit_view_mode', addr) +end + +if scan_all or not is_known 'ui_unit_view_mode' then + find_ui_unit_view_mode() +end