From 7eb4fc19de542db0d3e271123f24773e0c8c481e Mon Sep 17 00:00:00 2001 From: Alexander Gavrilov Date: Thu, 14 Jun 2012 12:46:12 +0400 Subject: [PATCH 01/28] Make dfhack.run_script usable from other scripts, and document it. --- LUA_API.rst | 12 ++++++++++++ Lua API.html | 10 ++++++++++ library/Core.cpp | 10 +++------- library/lua/dfhack.lua | 13 +++++++++---- 4 files changed, 34 insertions(+), 11 deletions(-) diff --git a/LUA_API.rst b/LUA_API.rst index f40b786d2..2859f8678 100644 --- a/LUA_API.rst +++ b/LUA_API.rst @@ -491,6 +491,13 @@ Currently it defines the following features: Compares to coroutine.resume like dfhack.safecall vs pcall. +* ``dfhack.run_script(name[,args...])`` + + Run a lua script in hack/scripts/, as if it was started from dfhack command-line. + The ``name`` argument should be the name stem, as would be used on the command line. + Note that the script is re-read from the file every time it is called, and errors + are propagated to the caller. + * ``dfhack.with_suspend(f[,args...])`` Calls ``f`` with arguments after grabbing the DF core suspend lock. @@ -1148,6 +1155,11 @@ Internal API These functions are intended for the use by dfhack developers, and are only documented here for completeness: +* ``dfhack.internal.scripts`` + + The table used by ``dfhack.run_script()`` to give every script its own + global environment, persistent between calls to the script. + * ``dfhack.internal.getAddress(name)`` Returns the global address ``name``, or *nil*. diff --git a/Lua API.html b/Lua API.html index 5574c55b8..047457985 100644 --- a/Lua API.html +++ b/Lua API.html @@ -766,6 +766,12 @@ returning. Intended as a convenience function.

  • dfhack.saferesume(coroutine[,args...])

    Compares to coroutine.resume like dfhack.safecall vs pcall.

    • +

  • dfhack.run_script(name[,args...])

    +

    Run a lua script in hack/scripts/, as if it was started from dfhack command-line. +The name argument should be the name stem, as would be used on the command line. +Note that the script is re-read from the file every time it is called, and errors +are propagated to the caller.

    +

  • dfhack.with_suspend(f[,args...])

    Calls f with arguments after grabbing the DF core suspend lock. Suspending is necessary for accessing a consistent state of DF memory.

    @@ -1310,6 +1316,10 @@ Returns true, was_only_planned if removed; or false if none fo

    These functions are intended for the use by dfhack developers, and are only documented here for completeness:

    diff --git a/library/LuaApi.cpp b/library/LuaApi.cpp index 59956a2c8..50458f119 100644 --- a/library/LuaApi.cpp +++ b/library/LuaApi.cpp @@ -1021,6 +1021,21 @@ static const luaL_Reg dfhack_constructions_funcs[] = { /***** Internal module *****/ +static void *checkaddr(lua_State *L, int idx, bool allow_null = false) +{ + luaL_checkany(L, idx); + void *rv; + if (lua_isnil(L, idx)) + rv = NULL; + else if (lua_type(L, idx) == LUA_TNUMBER) + rv = (void*)lua_tointeger(L, idx); + else + rv = Lua::CheckDFObject(L, NULL, idx); + if (!rv && !allow_null) + luaL_argerror(L, idx, "null pointer"); + return rv; +} + static uint32_t getBase() { return Core::getInstance().p->getBase(); } static const LuaWrapper::FunctionReg dfhack_internal_module[] = { @@ -1042,7 +1057,7 @@ static int internal_getAddress(lua_State *L) static int internal_setAddress(lua_State *L) { std::string name = luaL_checkstring(L, 1); - uint32_t addr = luaL_checkint(L, 2); + uint32_t addr = (uint32_t)checkaddr(L, 2, true); internal_getAddress(L); // Set the address @@ -1097,10 +1112,99 @@ static int internal_getMemRanges(lua_State *L) return 1; } +static int internal_memmove(lua_State *L) +{ + void *dest = checkaddr(L, 1); + void *src = checkaddr(L, 2); + int size = luaL_checkint(L, 3); + if (size < 0) luaL_argerror(L, 1, "negative size"); + memmove(dest, src, size); + return 0; +} + +static int internal_memcmp(lua_State *L) +{ + void *dest = checkaddr(L, 1); + void *src = checkaddr(L, 2); + int size = luaL_checkint(L, 3); + if (size < 0) luaL_argerror(L, 1, "negative size"); + lua_pushinteger(L, memcmp(dest, src, size)); + return 1; +} + +static int internal_memscan(lua_State *L) +{ + uint8_t *haystack = (uint8_t*)checkaddr(L, 1); + int hcount = luaL_checkint(L, 2); + int hstep = luaL_checkint(L, 3); + if (hstep == 0) luaL_argerror(L, 3, "zero step"); + void *needle = checkaddr(L, 4); + int nsize = luaL_checkint(L, 5); + if (nsize < 0) luaL_argerror(L, 5, "negative size"); + + for (int i = 0; i <= hcount; i++) + { + uint8_t *p = haystack + i*hstep; + if (memcmp(p, needle, nsize) == 0) { + lua_pushinteger(L, i); + lua_pushinteger(L, (lua_Integer)p); + return 2; + } + } + + lua_pushnil(L); + return 1; +} + +static int internal_diffscan(lua_State *L) +{ + lua_settop(L, 8); + void *old_data = checkaddr(L, 1); + void *new_data = checkaddr(L, 2); + int start_idx = luaL_checkint(L, 3); + int end_idx = luaL_checkint(L, 4); + int eltsize = luaL_checkint(L, 5); + bool has_oldv = !lua_isnil(L, 6); + bool has_newv = !lua_isnil(L, 7); + bool has_diffv = !lua_isnil(L, 8); + +#define LOOP(esz, etype) \ + case esz: { \ + etype *pold = (etype*)old_data; \ + etype *pnew = (etype*)new_data; \ + etype oldv = (etype)luaL_optint(L, 6, 0); \ + etype newv = (etype)luaL_optint(L, 7, 0); \ + etype diffv = (etype)luaL_optint(L, 8, 0); \ + for (int i = start_idx; i < end_idx; i++) { \ + if (pold[i] == pnew[i]) continue; \ + if (has_oldv && pold[i] != oldv) continue; \ + if (has_newv && pnew[i] != newv) continue; \ + if (has_diffv && etype(pnew[i]-pold[i]) != diffv) continue; \ + lua_pushinteger(L, i); return 1; \ + } \ + break; \ + } + switch (eltsize) { + LOOP(1, uint8_t); + LOOP(2, uint16_t); + LOOP(4, uint32_t); + default: + luaL_argerror(L, 5, "invalid element size"); + } +#undef LOOP + + lua_pushnil(L); + return 1; +} + static const luaL_Reg dfhack_internal_funcs[] = { { "getAddress", internal_getAddress }, { "setAddress", internal_setAddress }, { "getMemRanges", internal_getMemRanges }, + { "memmove", internal_memmove }, + { "memcmp", internal_memcmp }, + { "memscan", internal_memscan }, + { "diffscan", internal_diffscan }, { NULL, NULL } }; From db91850464a98e785ff53cfdc5df46b10229cfdb Mon Sep 17 00:00:00 2001 From: Alexander Gavrilov Date: Sat, 16 Jun 2012 14:42:56 +0400 Subject: [PATCH 09/28] Sync to the change in gamemode/gametype globals. --- library/RemoteTools.cpp | 16 +++++------ library/include/modules/World.h | 31 +++++---------------- library/modules/World.cpp | 10 +++---- library/xml | 2 +- plugins/burrows.cpp | 4 +-- plugins/mode.cpp | 48 ++++++++++++++++----------------- plugins/reveal.cpp | 10 +++---- plugins/seedwatch.cpp | 4 +-- 8 files changed, 54 insertions(+), 71 deletions(-) diff --git a/library/RemoteTools.cpp b/library/RemoteTools.cpp index 689c783a8..95c495e93 100644 --- a/library/RemoteTools.cpp +++ b/library/RemoteTools.cpp @@ -379,19 +379,19 @@ static command_result GetWorldInfo(color_ostream &stream, if (!ui || !world || !Core::getInstance().isWorldLoaded()) return CR_NOT_FOUND; - t_gamemodes mode; - if (!Core::getInstance().getWorld()->ReadGameMode(mode)) - mode.g_type = GAMETYPE_DWARF_MAIN; + df::game_type gt = game_type::DWARF_MAIN; + if (df::global::gametype) + gt = *df::global::gametype; out->set_save_dir(world->cur_savegame.save_dir); if (world->world_data->name.has_name) describeName(out->mutable_world_name(), &world->world_data->name); - switch (mode.g_type) + switch (gt) { - case GAMETYPE_DWARF_MAIN: - case GAMETYPE_DWARF_RECLAIM: + case game_type::DWARF_MAIN: + case game_type::DWARF_RECLAIM: out->set_mode(GetWorldInfoOut::MODE_DWARF); out->set_civ_id(ui->civ_id); out->set_site_id(ui->site_id); @@ -399,7 +399,7 @@ static command_result GetWorldInfo(color_ostream &stream, out->set_race_id(ui->race_id); break; - case GAMETYPE_ADVENTURE_MAIN: + case game_type::ADVENTURE_MAIN: out->set_mode(GetWorldInfoOut::MODE_ADVENTURE); if (auto unit = vector_get(world->units.active, 0)) @@ -423,7 +423,7 @@ static command_result GetWorldInfo(color_ostream &stream, } break; - case GAMETYPE_VIEW_LEGENDS: + case game_type::VIEW_LEGENDS: out->set_mode(GetWorldInfoOut::MODE_LEGENDS); break; diff --git a/library/include/modules/World.h b/library/include/modules/World.h index 81ebfbd60..8b0fe3f59 100644 --- a/library/include/modules/World.h +++ b/library/include/modules/World.h @@ -35,32 +35,15 @@ distribution. #include "Module.h" #include +#include "DataDefs.h" + namespace DFHack { - /** - * \ingroup grp_world - */ - enum GameMode - { - GAMEMODE_DWARF, - GAMEMODE_ADVENTURE, - GAMEMODENUM, - GAMEMODE_NONE - }; - /** - * \ingroup grp_world - */ - enum GameType - { - GAMETYPE_DWARF_MAIN, - GAMETYPE_ADVENTURE_MAIN, - GAMETYPE_VIEW_LEGENDS, - GAMETYPE_DWARF_RECLAIM, - GAMETYPE_DWARF_ARENA, - GAMETYPE_ADVENTURE_ARENA, - GAMETYPENUM, - GAMETYPE_NONE - }; + typedef df::game_mode GameMode; + typedef df::game_type GameType; + +#define GAMEMODE_ADVENTURE df::enums::game_mode::ADVENTURE + /** * \ingroup grp_world */ diff --git a/library/modules/World.cpp b/library/modules/World.cpp index 4b50b44ca..393e7cbfe 100644 --- a/library/modules/World.cpp +++ b/library/modules/World.cpp @@ -85,7 +85,7 @@ World::World() if(df::global::current_weather) d->StartedWeather = true; - if (df::global::game_mode && df::global::control_mode) + if (df::global::gamemode && df::global::gametype) d->StartedMode = true; d->Inited = true; @@ -132,8 +132,8 @@ bool World::ReadGameMode(t_gamemodes& rd) { if(d->Inited && d->StartedMode) { - rd.g_mode = (DFHack::GameMode)*df::global::control_mode; - rd.g_type = (DFHack::GameType)*df::global::game_mode; + rd.g_mode = (DFHack::GameMode)*df::global::gamemode; + rd.g_type = (DFHack::GameType)*df::global::gametype; return true; } return false; @@ -142,8 +142,8 @@ bool World::WriteGameMode(const t_gamemodes & wr) { if(d->Inited && d->StartedMode) { - *df::global::control_mode = wr.g_mode; - *df::global::game_mode = wr.g_type; + *df::global::gamemode = wr.g_mode; + *df::global::gametype = wr.g_type; return true; } return false; diff --git a/library/xml b/library/xml index c38188466..bc757db69 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit c381884664c71adefbec44258a734def2c88dacc +Subproject commit bc757db69514b54eb66d4d38e9cc1354d3761907 diff --git a/plugins/burrows.cpp b/plugins/burrows.cpp index 9ff902c3d..0b66a7b9a 100644 --- a/plugins/burrows.cpp +++ b/plugins/burrows.cpp @@ -101,8 +101,8 @@ DFhackCExport command_result plugin_onstatechange(color_ostream &out, state_chan switch (event) { case SC_MAP_LOADED: deinit_map(out); - if (df::global::game_mode && - *df::global::game_mode == GAMEMODE_DWARF) + if (df::global::gamemode && + *df::global::gamemode == game_mode::DWARF) init_map(out); break; case SC_MAP_UNLOADED: diff --git a/plugins/mode.cpp b/plugins/mode.cpp index 33dd5875f..f9e6cd10c 100644 --- a/plugins/mode.cpp +++ b/plugins/mode.cpp @@ -10,7 +10,7 @@ using namespace std; #include "modules/World.h" #include using namespace DFHack; - +using namespace df::enums; command_result mode (color_ostream &out, vector & parameters); @@ -44,28 +44,28 @@ void printCurrentModes(t_gamemodes gm, Console & con) con << "Current game type:\t" << gm.g_type << " ("; switch(gm.g_type) { - case GAMETYPE_DWARF_MAIN: + case game_type::DWARF_MAIN: con << "Fortress)" << endl; break; - case GAMETYPE_ADVENTURE_MAIN: + case game_type::ADVENTURE_MAIN: con << "Adventurer)" << endl; break; - case GAMETYPE_VIEW_LEGENDS: + case game_type::VIEW_LEGENDS: con << "Legends)" << endl; break; - case GAMETYPE_DWARF_RECLAIM: + case game_type::DWARF_RECLAIM: con << "Reclaim)" << endl; break; - case GAMETYPE_DWARF_ARENA: + case game_type::DWARF_ARENA: con << "Arena)" << endl; break; - case GAMETYPE_ADVENTURE_ARENA: + case game_type::ADVENTURE_ARENA: con << "Arena - control creature)" << endl; break; - case GAMETYPENUM: + case game_type::num: con << "INVALID)" << endl; break; - case GAMETYPE_NONE: + case game_type::NONE: con << "NONE)" << endl; break; default: @@ -75,16 +75,16 @@ void printCurrentModes(t_gamemodes gm, Console & con) con << "Current game mode:\t" << gm.g_mode << " ("; switch (gm.g_mode) { - case GAMEMODE_DWARF: + case game_mode::DWARF: con << "Dwarf)" << endl; break; - case GAMEMODE_ADVENTURE: + case game_mode::ADVENTURE: con << "Adventure)" << endl; break; - case GAMEMODENUM: + case game_mode::num: con << "INVALID)" << endl; break; - case GAMEMODE_NONE: + case game_mode::NONE: con << "NONE)" << endl; break; default: @@ -132,7 +132,7 @@ command_result mode (color_ostream &out_, vector & parameters) { if(!abuse) { - if( gm.g_mode == GAMEMODE_NONE || gm.g_type == GAMETYPE_VIEW_LEGENDS) + if( gm.g_mode == game_mode::NONE || gm.g_type == game_type::VIEW_LEGENDS) { out.printerr("It is not safe to set modes in menus.\n"); return CR_FAILURE; @@ -163,24 +163,24 @@ command_result mode (color_ostream &out_, vector & parameters) switch(select) { case 0: - gm.g_mode = GAMEMODE_DWARF; - gm.g_type = GAMETYPE_DWARF_MAIN; + gm.g_mode = game_mode::DWARF; + gm.g_type = game_type::DWARF_MAIN; break; case 1: - gm.g_mode = GAMEMODE_ADVENTURE; - gm.g_type = GAMETYPE_ADVENTURE_MAIN; + gm.g_mode = game_mode::ADVENTURE; + gm.g_type = game_type::ADVENTURE_MAIN; break; case 2: - gm.g_mode = GAMEMODE_DWARF; - gm.g_type = GAMETYPE_DWARF_ARENA; + gm.g_mode = game_mode::DWARF; + gm.g_type = game_type::DWARF_ARENA; break; case 3: - gm.g_mode = GAMEMODE_ADVENTURE; - gm.g_type = GAMETYPE_ADVENTURE_ARENA; + gm.g_mode = game_mode::ADVENTURE; + gm.g_type = game_type::ADVENTURE_ARENA; break; case 4: - gm.g_mode = GAMEMODE_DWARF; - gm.g_type = GAMETYPE_DWARF_RECLAIM; + gm.g_mode = game_mode::DWARF; + gm.g_type = game_type::DWARF_RECLAIM; break; } } diff --git a/plugins/reveal.cpp b/plugins/reveal.cpp index 513eededb..f35424309 100644 --- a/plugins/reveal.cpp +++ b/plugins/reveal.cpp @@ -90,7 +90,7 @@ DFhackCExport command_result plugin_onupdate ( color_ostream &out ) World *World = Core::getInstance().getWorld(); t_gamemodes gm; World->ReadGameMode(gm); - if(gm.g_mode == GAMEMODE_DWARF) + if(gm.g_mode == game_mode::DWARF) { // if the map is revealed and we're in fortress mode, force the game to pause. if(revealed == REVEALED) @@ -193,12 +193,12 @@ command_result reveal(color_ostream &out, vector & params) } t_gamemodes gm; World->ReadGameMode(gm); - if(gm.g_mode == GAMEMODE_ADVENTURE) + if(gm.g_mode == game_mode::ADVENTURE) { revealAdventure(out); return CR_OK; } - if(gm.g_mode != GAMEMODE_DWARF) + if(gm.g_mode != game_mode::DWARF) { con.printerr("Only in fortress mode.\n"); return CR_FAILURE; @@ -272,7 +272,7 @@ command_result unreveal(color_ostream &out, vector & params) } t_gamemodes gm; World->ReadGameMode(gm); - if(gm.g_mode != GAMEMODE_DWARF) + if(gm.g_mode != game_mode::DWARF) { con.printerr("Only in fortress mode.\n"); return CR_FAILURE; @@ -350,7 +350,7 @@ command_result revflood(color_ostream &out, vector & params) } t_gamemodes gm; World->ReadGameMode(gm); - if(gm.g_type != GAMETYPE_DWARF_MAIN && gm.g_mode != GAMEMODE_DWARF ) + if(gm.g_type != game_type::DWARF_MAIN && gm.g_mode != game_mode::DWARF ) { out.printerr("Only in proper dwarf mode.\n"); return CR_FAILURE; diff --git a/plugins/seedwatch.cpp b/plugins/seedwatch.cpp index b6097f628..ae9ff0e14 100755 --- a/plugins/seedwatch.cpp +++ b/plugins/seedwatch.cpp @@ -111,7 +111,7 @@ command_result df_seedwatch(color_ostream &out, vector& parameters) w->ReadGameMode(gm);// FIXME: check return value // if game mode isn't fortress mode - if(gm.g_mode != GAMEMODE_DWARF || gm.g_type != GAMETYPE_DWARF_MAIN) + if(gm.g_mode != game_mode::DWARF || gm.g_type != game_type::DWARF_MAIN) { // just print the help printHelp(out); @@ -299,7 +299,7 @@ DFhackCExport command_result plugin_onupdate(color_ostream &out) t_gamemodes gm; w->ReadGameMode(gm);// FIXME: check return value // if game mode isn't fortress mode - if(gm.g_mode != GAMEMODE_DWARF || gm.g_type != GAMETYPE_DWARF_MAIN) + if(gm.g_mode != game_mode::DWARF || gm.g_type != game_type::DWARF_MAIN) { // stop running. running = false; From 927ce6ce5adf019fd4f5fe18024b4f1b4c17fc20 Mon Sep 17 00:00:00 2001 From: Alexander Gavrilov Date: Sat, 16 Jun 2012 17:09:58 +0400 Subject: [PATCH 10/28] Fix a problem with number to address cast for high-half addresses. If the address is out of the signed int range, lua_tointeger produces unspecified result. lua_tounsigned is guaranteed to wrap. --- library/LuaApi.cpp | 2 +- library/LuaWrapper.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/library/LuaApi.cpp b/library/LuaApi.cpp index 50458f119..cdfd47892 100644 --- a/library/LuaApi.cpp +++ b/library/LuaApi.cpp @@ -1028,7 +1028,7 @@ static void *checkaddr(lua_State *L, int idx, bool allow_null = false) if (lua_isnil(L, idx)) rv = NULL; else if (lua_type(L, idx) == LUA_TNUMBER) - rv = (void*)lua_tointeger(L, idx); + rv = (void*)lua_tounsigned(L, idx); else rv = Lua::CheckDFObject(L, NULL, idx); if (!rv && !allow_null) diff --git a/library/LuaWrapper.cpp b/library/LuaWrapper.cpp index 7c15a27aa..9f2fc2f16 100644 --- a/library/LuaWrapper.cpp +++ b/library/LuaWrapper.cpp @@ -715,7 +715,7 @@ static int meta_reinterpret_cast(lua_State *state) if (lua_isnil(state, 2)) ptr = NULL; else if (lua_isnumber(state, 2)) - ptr = (void*)lua_tointeger(state, 2); + ptr = (void*)lua_tounsigned(state, 2); else { ptr = get_object_internal(state, NULL, 2, false, true); From 67536da2fe718b103f4200b95f00cd30000cb7ad Mon Sep 17 00:00:00 2001 From: Alexander Gavrilov Date: Sat, 16 Jun 2012 19:51:15 +0400 Subject: [PATCH 11/28] Add an interactive script finding a limited subset of linux offsets. --- library/lua/memscan.lua | 419 +++++++++++++++++++++++++++++++++ library/lua/utils.lua | 44 ++++ scripts/devel/find-offsets.lua | 368 +++++++++++++++++++++++++++++ 3 files changed, 831 insertions(+) create mode 100644 library/lua/memscan.lua create mode 100644 scripts/devel/find-offsets.lua diff --git a/library/lua/memscan.lua b/library/lua/memscan.lua new file mode 100644 index 000000000..e5a0c6f72 --- /dev/null +++ b/library/lua/memscan.lua @@ -0,0 +1,419 @@ +-- Utilities for offset scan scripts. + +local _ENV = mkmodule('memscan') + +local utils = require('utils') + +-- A length-checked view on a memory buffer + +CheckedArray = CheckedArray or {} + +function CheckedArray.new(type,saddr,eaddr) + local data = df.reinterpret_cast(type,saddr) + local esize = data:sizeof() + local count = math.floor((eaddr-saddr)/esize) + local obj = { + type = type, start = saddr, size = count*esize, + esize = esize, data = data, count = count + } + setmetatable(obj, CheckedArray) + return obj +end + +function CheckedArray:__len() + return self.count +end +function CheckedArray:__index(idx) + if type(idx) == number then + if idx >= self.count then + error('Index out of bounds: '..tostring(idx)) + end + return self.data[idx] + else + return CheckedArray[idx] + end +end +function CheckedArray:__newindex(idx, val) + if idx >= self.count then + error('Index out of bounds: '..tostring(idx)) + end + self.data[idx] = val +end +function CheckedArray:addr2idx(addr, round) + local off = addr - self.start + if off >= 0 and off < self.size and (round or (off % self.esize) == 0) then + return math.floor(off / self.esize), off + end +end +function CheckedArray:idx2addr(idx) + if idx >= 0 and idx < self.count then + return self.start + idx*self.esize + end +end + +-- Search methods + +function CheckedArray:find(data,sidx,eidx,reverse) + local dcnt = #data + sidx = math.max(0, sidx or 0) + eidx = math.min(self.count, eidx or self.count) + if (eidx - sidx) >= dcnt and dcnt > 0 then + return dfhack.with_temp_object( + df.new(self.type, dcnt), + function(buffer) + for i = 1,dcnt do + buffer[i-1] = data[i] + end + local cnt = eidx - sidx - dcnt + local step = self.esize + local sptr = self.start + sidx*step + local ksize = dcnt*step + if reverse then + local idx, addr = dfhack.internal.memscan(sptr + cnt*step, cnt, -step, buffer, ksize) + if idx then + return sidx + cnt - idx, addr + end + else + local idx, addr = dfhack.internal.memscan(sptr, cnt, step, buffer, ksize) + if idx then + return sidx + idx, addr + end + end + end + ) + end +end +function CheckedArray:find_one(data,sidx,eidx,reverse) + local idx, addr = self:find(data,sidx,eidx,reverse) + if idx then + -- Verify this is the only match + if reverse then + eidx = idx+#data-1 + else + sidx = idx+1 + end + if self:find(data,sidx,eidx,reverse) then + return nil + end + end + return idx, addr +end +function CheckedArray:list_changes(old_arr,old_val,new_val,delta) + if old_arr.type ~= self.type or old_arr.count ~= self.count then + error('Incompatible arrays') + end + local eidx = self.count + local optr = old_arr.start + local nptr = self.start + local esize = self.esize + local rv + local sidx = 0 + while true do + local idx = dfhack.internal.diffscan(optr, nptr, sidx, eidx, esize, old_val, new_val, delta) + if not idx then + break + end + rv = rv or {} + rv[#rv+1] = idx + sidx = idx+1 + end + return rv +end +function CheckedArray:filter_changes(prev_list,old_arr,old_val,new_val,delta) + if old_arr.type ~= self.type or old_arr.count ~= self.count then + error('Incompatible arrays') + end + local eidx = self.count + local optr = old_arr.start + local nptr = self.start + local esize = self.esize + local rv + for i=1,#prev_list do + local idx = prev_list[i] + if idx < 0 or idx >= eidx then + error('Index out of bounds: '..idx) + end + if dfhack.internal.diffscan(optr, nptr, idx, idx+1, esize, old_val, new_val, delta) then + rv = rv or {} + rv[#rv+1] = idx + end + end + return rv +end + +-- A raw memory area class + +MemoryArea = MemoryArea or {} +MemoryArea.__index = MemoryArea + +function MemoryArea.new(astart, aend) + local obj = { + start_addr = astart, end_addr = aend, size = aend - astart, + int8_t = CheckedArray.new('int8_t',astart,aend), + uint8_t = CheckedArray.new('uint8_t',astart,aend), + int16_t = CheckedArray.new('int16_t',astart,aend), + uint16_t = CheckedArray.new('uint16_t',astart,aend), + int32_t = CheckedArray.new('int32_t',astart,aend), + uint32_t = CheckedArray.new('uint32_t',astart,aend) + } + setmetatable(obj, MemoryArea) + return obj +end + +function MemoryArea:__gc() + df.delete(self.buffer) +end + +function MemoryArea:__tostring() + return string.format('', self.start_addr, self.end_addr) +end +function MemoryArea:contains_range(start,size) + return start >= self.start_addr and (start+size) <= self.end_addr +end +function MemoryArea:contains_obj(obj,count) + local size, base = df.sizeof(obj) + return size and base and self:contains_range(base, size*(count or 1)) +end + +function MemoryArea:clone() + local buffer = df.new('int8_t', self.size) + local _, base = buffer:sizeof() + local rv = MemoryArea.new(base, base+self.size) + rv.buffer = buffer + return rv +end +function MemoryArea:copy_from(area2) + if area2.size ~= self.size then + error('Size mismatch') + end + dfhack.internal.memmove(self.start_addr, area2.start_addr, self.size) +end +function MemoryArea:delete() + setmetatable(self, nil) + df.delete(self.buffer) + for k,v in pairs(self) do self[k] = nil end +end + +-- Static data segment search + +local function find_data_segment() + local data_start, data_end + + for i,mem in ipairs(dfhack.internal.getMemRanges()) do + if data_end then + if mem.start_addr == data_end and mem.read and mem.write then + data_end = mem.end_addr + else + break + end + elseif mem.read and mem.write + and (string.match(mem.name,'/dwarfort%.exe$') + or string.match(mem.name,'/Dwarf_Fortress$')) + then + data_start = mem.start_addr + data_end = mem.end_addr + end + end + + return data_start, data_end +end + +function get_data_segment() + local s, e = find_data_segment() + if s and e then + return MemoryArea.new(s, e) + end +end + +-- Register a found offset, or report an error. + +function found_offset(name,val) + local cval = dfhack.internal.getAddress(name) + + if not val then + print('Could not find offset '..name) + if not cval and not utils.prompt_yes_no('Continue with the script?') then + error('User quit') + end + return + end + + if df.isvalid(val) then + _,val = val:sizeof() + end + + print(string.format('Found offset %s: %x', name, val)) + + if cval then + if cval ~= val then + error(string.format('Mismatch with the current value: %x',val)) + end + else + dfhack.internal.setAddress(name, val) + end +end + +-- Offset of a field within struct + +function field_ref(handle,...) + local items = table.pack(...) + for i=1,items.n-1 do + handle = handle[items[i]] + end + return handle:_field(items[items.n]) +end + +function field_offset(type,...) + local handle = df.reinterpret_cast(type,1) + local _,addr = df.sizeof(field_ref(handle,...)) + return addr-1 +end + +function MemoryArea:object_by_field(addr,type,...) + if not addr then + return nil + end + local base = addr - field_offset(type,...) + local obj = df.reinterpret_cast(type, base) + if not self:contains_obj(obj) then + obj = nil + end + return obj, base +end + +-- Validation + +function is_valid_vector(ref,size) + local ints = df.reinterpret_cast('uint32_t', ref) + return ints[0] <= ints[1] and ints[1] <= ints[2] + and (size == nil or (ints[1] - ints[0]) % size == 0) +end + +-- Difference search helper + +DiffSearcher = DiffSearcher or {} +DiffSearcher.__index = DiffSearcher + +function DiffSearcher.new(area) + local obj = { area = area } + setmetatable(obj, DiffSearcher) + return obj +end + +function DiffSearcher:begin_search(type) + self.type = type + self.old_value = nil + self.search_sets = nil + if not self.save_area then + self.save_area = self.area:clone() + end +end +function DiffSearcher:advance_search(new_value,delta) + if self.search_sets then + local nsets = #self.search_sets + local ovec = self.save_area[self.type] + local nvec = self.area[self.type] + local new_set + if nsets > 0 then + local last_set = self.search_sets[nsets] + new_set = nvec:filter_changes(last_set,ovec,self.old_value,new_value,delta) + else + new_set = nvec:list_changes(ovec,self.old_value,new_value,delta) + end + if new_set then + self.search_sets[nsets+1] = new_set + self.old_value = new_value + self.save_area:copy_from(self.area) + return #new_set, new_set + end + else + self.old_value = new_value + self.search_sets = {} + self.save_area:copy_from(self.area) + return #self.save_area[self.type], nil + end +end +function DiffSearcher:reset() + self.search_sets = nil + if self.save_area then + self.save_area:delete() + self.save_area = nil + end +end +function DiffSearcher:idx2addr(idx) + return self.area[self.type]:idx2addr(idx) +end + +-- Menu search utility + +function find_menu_cursor(searcher,prompt,data_type,choices,enum) + enum = enum or {} + + -- Loop for restarting search from scratch + while true do + print('\n'..prompt) + + searcher:begin_search(data_type) + + local found = false + local ccursor = 0 + + -- Loop through choices + while true do + local choice + + -- Select the next value to search for + if type(choices) == 'function' then + print('') + + choice = choices(ccursor) + ccursor = ccursor + 1 + + if not choice then + break + end + else + choice = choices[ccursor+1] + ccursor = (ccursor+1) % #choices + + local cname = enum[choice] or choice + if type(choice) == 'string' and type(cname) == 'number' then + choice, cname = cname, choice + end + if cname ~= choice then + cname = cname..' ('..choice..')' + end + + -- Ask the user to select it + print('\n Please select: '..cname) + if not utils.prompt_yes_no(' Continue?', true) then + break + end + end + + -- Search for it in the memory + local cnt, set = searcher:advance_search(choice) + if not cnt then + dfhack.printerr(' Converged to zero candidates; probably a mistake somewhere.') + break + elseif set and cnt == 1 then + -- To confirm, wait for two 1-candidate results in a row + if found then + local addr = searcher:idx2addr(set[1]) + print(string.format(' Confirmed address: %x\n', addr)) + return addr, set[1] + else + found = true + end + end + + print(' '..cnt..' candidates remaining.') + end + + if not utils.prompt_yes_no('\nRetry search from the start?') then + return nil + end + end +end + +return _ENV diff --git a/library/lua/utils.lua b/library/lua/utils.lua index e67801f4f..4fac56ece 100644 --- a/library/lua/utils.lua +++ b/library/lua/utils.lua @@ -361,4 +361,48 @@ function insert_or_update(vector,item,field,cmp) return added,cur,pos end +-- Ask a yes-no question +function prompt_yes_no(msg,default) + local prompt = msg + if default == nil then + prompt = prompt..' (y/n): ' + elseif default then + prompt = prompt..' (y/n/enter=y): ' + else + prompt = prompt..' (y/n/enter=n): ' + end + while true do + local rv = dfhack.lineedit(prompt) + if rv then + if string.match(rv,'^[Yy]') then + return true + elseif string.match(rv,'^[Nn]') then + return false + elseif rv == '' and default ~= nil then + return default + end + end + end +end + +-- Ask for input with check function +function prompt_input(prompt,check,quit_str) + quit_str = quit_str or '~~~' + while true do + local rv = dfhack.lineedit(prompt) + if rv == quit_str then + return nil + end + local rtbl = table.pack(check(rv)) + if rtbl[1] then + return table.unpack(rtbl,2,rtbl.n) + end + end +end + +function check_number(text) + local nv = tonumber(text) + return nv ~= nil, nv +end + return _ENV \ No newline at end of file diff --git a/scripts/devel/find-offsets.lua b/scripts/devel/find-offsets.lua new file mode 100644 index 000000000..ef16026e3 --- /dev/null +++ b/scripts/devel/find-offsets.lua @@ -0,0 +1,368 @@ +-- Find some offsets for linux. + +local utils = require 'utils' +local ms = require 'memscan' + +local scan_all = false +local is_known = dfhack.internal.getAddress + +collectgarbage() + +print[[ +WARNING: THIS SCRIPT IS STRICTLY FOR DFHACK DEVELOPERS. + +Running this script on a new DF version will NOT +MAKE IT RUN CORRECTLY if any data structures +changed, thus possibly leading to CRASHES AND/OR +PERMANENT SAVE CORRUPTION. + +This script should be initially started immediately +after loading the game, WITHOUT first loading a world. +It expects vanilla game configuration, without any +custom tilesets or init file changes. +]] + +if not utils.prompt_yes_no('Proceed?') then + return +end + +-- Data segment location + +local data = ms.get_data_segment() +if not data then + error('Could not find data segment') +end + +print('Data section: '..tostring(data)) +if data.size < 5000000 then + error('Data segment too short.') +end + +local searcher = ms.DiffSearcher.new(data) + +local function validate_offset(name,validator,addr,tname,...) + local obj = data:object_by_field(addr,tname,...) + if obj and not validator(obj) then + obj = nil + end + ms.found_offset(name,obj) +end + +-- +-- Cursor group +-- + +local function find_cursor() + print('\nPlease navigate to the title screen to find cursor.') + if not utils.prompt_yes_no('Proceed?', true) then + return false + end + + -- Unpadded version + local idx, addr = data.int32_t:find_one{ + -30000, -30000, -30000, + -30000, -30000, -30000, -30000, -30000, -30000, + df.game_mode.NONE, df.game_type.NONE + } + if idx then + ms.found_offset('cursor', addr) + ms.found_offset('selection_rect', addr + 12) + ms.found_offset('gamemode', addr + 12 + 24) + ms.found_offset('gametype', addr + 12 + 24 + 4) + return true + end + + -- Padded version + idx, addr = data.int32_t:find_one{ + -30000, -30000, -30000, 0, + -30000, -30000, -30000, -30000, -30000, -30000, 0, 0, + df.game_mode.NONE, 0, 0, 0, df.game_type.NONE + } + if idx then + ms.found_offset('cursor', addr) + ms.found_offset('selection_rect', addr + 0x10) + ms.found_offset('gamemode', addr + 0x30) + ms.found_offset('gametype', addr + 0x40) + return true + end + + dfhack.printerr('Could not find cursor.') + return false +end + +if scan_all or not ( + is_known 'cursor' and is_known 'selection_rect' and + is_known 'gamemode' and is_known 'gametype' +) then + find_cursor() +end + +-- +-- Announcements +-- + +local function find_announcements() + idx, addr = data.int32_t:find_one{ + 25, 25, 31, 31, 24, 24, 40, 40, 40, 40, 40, 40, 40 + } + if idx then + ms.found_offset('announcements', addr) + return + end + + dfhack.printerr('Could not find announcements.') +end + +if scan_all or not is_known 'announcements' then + find_announcements() +end + +-- +-- d_init +-- + +local function is_valid_d_init(di) + if di.sky_tile ~= 178 then + print('Sky tile expected 178, found: '..di.sky_tile) + if not utils.prompt_yes_no('Ignore?') then + return false + end + end + + local ann = is_known 'announcements' + local size,ptr = di:sizeof() + if ann and ptr+size ~= ann then + print('Announcements not immediately after d_init.') + if not utils.prompt_yes_no('Ignore?') then + return false + end + end + + return true +end + +local function find_d_init() + idx, addr = data.int16_t:find_one{ + 1,0, 2,0, 5,0, 25,0, -- path_cost + 4,4, -- embark_rect + 20,1000,1000,1000,1000 -- store_dist + } + if idx then + validate_offset('d_init', is_valid_d_init, addr, df.d_init, 'path_cost') + return + end + + dfhack.printerr('Could not find d_init') +end + +if scan_all or not is_known 'd_init' then + find_d_init() +end + +-- +-- World +-- + +local function is_valid_world(world) + if not ms.is_valid_vector(world.units.all, 4) + or not ms.is_valid_vector(world.units.bad, 4) + or not ms.is_valid_vector(world.history.figures, 4) + or not ms.is_valid_vector(world.cur_savegame.map_features, 4) + then + dfhack.printerr('Vector layout check failed.') + return false + end + + if #world.units.all == 0 or #world.units.all ~= #world.units.bad then + print('Different or zero size of units.all and units.bad:'..#world.units.all..' vs '..#world.units.bad) + if not utils.prompt_yes_no('Ignore?') then + return false + end + end + + return true +end + +local function find_world() + local addr = ms.find_menu_cursor( + searcher, [[ +Searching for world. Please open the stockpile creation +menu, and follow instructions below:]], + 'int32_t', + { 'Corpses', 'Refuse', 'Stone', 'Wood', 'Gems', 'Bars', 'Cloth', 'Leather', 'Ammo', 'Coins' }, + df.stockpile_category + ) + validate_offset('world', is_valid_world, addr, df.world, 'selected_stockpile_type') +end + +if scan_all or not is_known 'world' then + find_world() +end + +-- +-- UI +-- + +local function is_valid_ui(ui) + if not ms.is_valid_vector(ui.economic_stone, 1) + or not ms.is_valid_vector(ui.dipscripts, 4) + then + dfhack.printerr('Vector layout check failed.') + return false + end + + if ui.follow_item ~= -1 or ui.follow_unit ~= -1 then + print('Invalid follow state: '..ui.follow_item..', '..ui.follow_unit) + return false + end + + return true +end + +local function find_ui() + local addr = ms.find_menu_cursor( + searcher, [[ +Searching for ui. Please open the designation +menu, and follow instructions below:]], + 'int16_t', + { 'DesignateMine', 'DesignateChannel', 'DesignateRemoveRamps', 'DesignateUpStair', + 'DesignateDownStair', 'DesignateUpDownStair', 'DesignateUpRamp', 'DesignateChopTrees' }, + df.ui_sidebar_mode + ) + validate_offset('ui', is_valid_ui, addr, df.ui, 'main', 'mode') +end + +if scan_all or not is_known 'ui' then + find_ui() +end + +-- +-- ui_sidebar_menus +-- + +local function is_valid_ui_sidebar_menus(usm) + if not ms.is_valid_vector(usm.workshop_job.choices_all, 4) + or not ms.is_valid_vector(usm.workshop_job.choices_visible, 4) + then + dfhack.printerr('Vector layout check failed.') + return false + end + + if #usm.workshop_job.choices_all == 0 + or #usm.workshop_job.choices_all ~= #usm.workshop_job.choices_visible then + print('Different or zero size of visible and all choices:'.. + #usm.workshop_job.choices_all..' vs '..#usm.workshop_job.choices_visible) + if not utils.prompt_yes_no('Ignore?') then + return false + end + end + + return true +end + +local function find_ui_sidebar_menus() + local addr = ms.find_menu_cursor( + searcher, [[ +Searching for ui_sidebar_menus. Please open the add job +ui of Mason, Craftsdwarfs, or Carpenters workshop, and +select entries in the list:]], + 'int32_t', + { 0, 1, 2, 3, 4, 5, 6 } + ) + validate_offset('ui_sidebar_menus', is_valid_ui_sidebar_menus, + addr, df.ui_sidebar_menus, 'workshop_job', 'cursor') +end + +if scan_all or not is_known 'ui_sidebar_menus' then + find_ui_sidebar_menus() +end + +-- +-- ui_build_selector +-- + +local function is_valid_ui_build_selector(ubs) + if not ms.is_valid_vector(ubs.requirements, 4) + or not ms.is_valid_vector(ubs.choices, 4) + then + dfhack.printerr('Vector layout check failed.') + return false + end + + if ubs.building_type ~= df.building_type.Trap + or ubs.building_subtype ~= df.trap_type.PressurePlate then + print('Invalid building type and subtype:'..ubs.building_type..','..ubs.building_subtype) + return false + end + + return true +end + +local function find_ui_build_selector() + local addr = ms.find_menu_cursor( + searcher, [[ +Searching for ui_build_selector. Please start constructing +a pressure plate, and enable creatures. Then change the min +weight as requested, remembering that the ui truncates the +number, so when it shows "Min (5000df", it means 50000:]], + 'int32_t', + { 50000, 49000, 48000, 47000, 46000, 45000, 44000 } + ) + validate_offset('ui_build_selector', is_valid_ui_build_selector, + addr, df.ui_build_selector, 'plate_info', 'unit_min') +end + +if scan_all or not is_known 'ui_build_selector' then + find_ui_build_selector() +end + +-- +-- ui_selected_unit +-- + +local function find_ui_selected_unit() + if not is_known 'world' then + dfhack.printerr('Cannot find ui_selected_unit: no world') + return + end + + for i,unit in ipairs(df.global.world.units.active) do + dfhack.units.setNickname(unit, i) + end + + local addr = ms.find_menu_cursor( + searcher, [[ +Searching for ui_selected_unit. Please activate the 'v' +mode, point it at units, and enter their numeric nickname +into the prompts below:]], + 'int32_t', + function() + return utils.prompt_input(' Enter index: ', utils.check_number) + end + ) + ms.found_offset('ui_selected_unit', addr) +end + +if scan_all or not is_known 'ui_selected_unit' then + find_ui_selected_unit() +end + +-- +-- ui_unit_view_mode +-- + +local function find_ui_unit_view_mode() + local addr = ms.find_menu_cursor( + searcher, [[ +Searching for ui_unit_view_mode. Having selected a unit +with 'v', switch the pages as requested:]], + 'int32_t', + { 'General', 'Inventory', 'Preferences', 'Wounds' }, + df.ui_unit_view_mode.T_value + ) + ms.found_offset('ui_unit_view_mode', addr) +end + +if scan_all or not is_known 'ui_unit_view_mode' then + find_ui_unit_view_mode() +end From fa41a27f2643afe8c8b601aab3e2ab3b1403411d Mon Sep 17 00:00:00 2001 From: Alexander Gavrilov Date: Sun, 17 Jun 2012 14:26:27 +0400 Subject: [PATCH 12/28] Add an api function to get vtable address from version info. --- LUA_API.rst | 4 ++++ Lua API.html | 3 +++ library/LuaApi.cpp | 12 ++++++++++++ 3 files changed, 19 insertions(+) diff --git a/LUA_API.rst b/LUA_API.rst index f3d26eaca..1723711d3 100644 --- a/LUA_API.rst +++ b/LUA_API.rst @@ -1168,6 +1168,10 @@ and are only documented here for completeness: Sets the global address ``name``. Returns the value of ``getAddress`` before the change. +* ``dfhack.internal.getVTable(name)`` + + Returns the pre-extracted vtable address ``name``, or *nil*. + * ``dfhack.internal.getBase()`` Returns the base address of the process. diff --git a/Lua API.html b/Lua API.html index 90e612116..f1bdd17d2 100644 --- a/Lua API.html +++ b/Lua API.html @@ -1326,6 +1326,9 @@ global environment, persistent between calls to the script.

  • dfhack.internal.setAddress(name, value)

    Sets the global address name. Returns the value of getAddress before the change.

    • +

  • dfhack.internal.getVTable(name)

    +

    Returns the pre-extracted vtable address name, or nil.

    +

  • dfhack.internal.getBase()

    Returns the base address of the process.

    • diff --git a/library/LuaApi.cpp b/library/LuaApi.cpp index cdfd47892..631b3c499 100644 --- a/library/LuaApi.cpp +++ b/library/LuaApi.cpp @@ -1080,6 +1080,17 @@ static int internal_setAddress(lua_State *L) return 1; } +static int internal_getVTable(lua_State *L) +{ + const char *name = luaL_checkstring(L, 1); + uint32_t addr = (uint32_t)Core::getInstance().vinfo->getVTable(name); + if (addr) + lua_pushnumber(L, addr); + else + lua_pushnil(L); + return 1; +} + static int internal_getMemRanges(lua_State *L) { std::vector ranges; @@ -1200,6 +1211,7 @@ static int internal_diffscan(lua_State *L) static const luaL_Reg dfhack_internal_funcs[] = { { "getAddress", internal_getAddress }, { "setAddress", internal_setAddress }, + { "getVTable", internal_getVTable }, { "getMemRanges", internal_getMemRanges }, { "memmove", internal_memmove }, { "memcmp", internal_memcmp }, From 534367546d137998231ca95103c7ec0d260bdc13 Mon Sep 17 00:00:00 2001 From: Alexander Gavrilov Date: Sun, 17 Jun 2012 15:59:03 +0400 Subject: [PATCH 13/28] Update to Lua 5.2.1 --- depends/lua/include/lfunc.h | 5 +- depends/lua/include/lgc.h | 10 +- depends/lua/include/llimits.h | 17 +- depends/lua/include/lobject.h | 75 +++--- depends/lua/include/lparser.h | 6 +- depends/lua/include/lstate.h | 20 +- depends/lua/include/lstring.h | 17 +- depends/lua/include/lua.h | 8 +- depends/lua/include/luaconf.h | 125 ++++++---- depends/lua/include/lundump.h | 4 +- depends/lua/src/lapi.c | 20 +- depends/lua/src/lauxlib.c | 17 +- depends/lua/src/lbaselib.c | 3 +- depends/lua/src/lcorolib.c | 5 +- depends/lua/src/ldblib.c | 21 +- depends/lua/src/ldebug.c | 13 +- depends/lua/src/ldo.c | 32 ++- depends/lua/src/ldump.c | 7 +- depends/lua/src/lfunc.c | 20 +- depends/lua/src/lgc.c | 456 +++++++++++++++++++++------------- depends/lua/src/llex.c | 27 +- depends/lua/src/lmathlib.c | 12 +- depends/lua/src/lmem.c | 18 +- depends/lua/src/loadlib.c | 97 ++++---- depends/lua/src/lopcodes.c | 3 +- depends/lua/src/loslib.c | 17 +- depends/lua/src/lparser.c | 153 +++++++----- depends/lua/src/lstate.c | 42 +++- depends/lua/src/lstring.c | 115 +++++++-- depends/lua/src/lstrlib.c | 35 +-- depends/lua/src/ltable.c | 27 +- depends/lua/src/lua.c | 31 +-- depends/lua/src/lundump.c | 38 ++- depends/lua/src/lvm.c | 50 ++-- depends/lua/src/lzio.c | 4 +- 35 files changed, 935 insertions(+), 615 deletions(-) diff --git a/depends/lua/include/lfunc.h b/depends/lua/include/lfunc.h index da1892315..e236a717c 100644 --- a/depends/lua/include/lfunc.h +++ b/depends/lua/include/lfunc.h @@ -1,5 +1,5 @@ /* -** $Id: lfunc.h,v 2.6 2010/06/04 13:06:15 roberto Exp $ +** $Id: lfunc.h,v 2.8 2012/05/08 13:53:33 roberto Exp $ ** Auxiliary functions to manipulate prototypes and closures ** See Copyright Notice in lua.h */ @@ -20,12 +20,11 @@ LUAI_FUNC Proto *luaF_newproto (lua_State *L); LUAI_FUNC Closure *luaF_newCclosure (lua_State *L, int nelems); -LUAI_FUNC Closure *luaF_newLclosure (lua_State *L, Proto *p); +LUAI_FUNC Closure *luaF_newLclosure (lua_State *L, int nelems); LUAI_FUNC UpVal *luaF_newupval (lua_State *L); LUAI_FUNC UpVal *luaF_findupval (lua_State *L, StkId level); LUAI_FUNC void luaF_close (lua_State *L, StkId level); LUAI_FUNC void luaF_freeproto (lua_State *L, Proto *f); -LUAI_FUNC void luaF_freeclosure (lua_State *L, Closure *c); LUAI_FUNC void luaF_freeupval (lua_State *L, UpVal *uv); LUAI_FUNC const char *luaF_getlocalname (const Proto *func, int local_number, int pc); diff --git a/depends/lua/include/lgc.h b/depends/lua/include/lgc.h index aa5dfce21..bdd5cce6c 100644 --- a/depends/lua/include/lgc.h +++ b/depends/lua/include/lgc.h @@ -1,5 +1,5 @@ /* -** $Id: lgc.h,v 2.52 2011/10/03 17:54:25 roberto Exp $ +** $Id: lgc.h,v 2.56 2012/05/23 15:43:14 roberto Exp $ ** Garbage Collector ** See Copyright Notice in lua.h */ @@ -25,6 +25,14 @@ */ + +/* how much to allocate before next GC step */ +#if !defined(GCSTEPSIZE) +/* ~100 small strings */ +#define GCSTEPSIZE (cast_int(100 * sizeof(TString))) +#endif + + /* ** Possible states of the Garbage Collector */ diff --git a/depends/lua/include/llimits.h b/depends/lua/include/llimits.h index 48dc81f76..fc9de1a11 100644 --- a/depends/lua/include/llimits.h +++ b/depends/lua/include/llimits.h @@ -1,5 +1,5 @@ /* -** $Id: llimits.h,v 1.95 2011/12/06 16:58:36 roberto Exp $ +** $Id: llimits.h,v 1.99 2012/05/28 20:32:28 roberto Exp $ ** Limits, basic types, and some other `installation-dependent' definitions ** See Copyright Notice in lua.h */ @@ -31,6 +31,8 @@ typedef unsigned char lu_byte; #define MAX_LUMEM ((lu_mem)(~(lu_mem)0)-2) +#define MAX_LMEM ((l_mem) ((MAX_LUMEM >> 1) - 2)) + #define MAX_INT (INT_MAX-2) /* maximum value of an int (-2 for safety) */ @@ -209,31 +211,36 @@ typedef lu_int32 Instruction; #elif defined(LUA_IEEE754TRICK) /* }{ */ /* the next trick should work on any machine using IEEE754 with - a 32-bit integer type */ + a 32-bit int type */ union luai_Cast { double l_d; LUA_INT32 l_p[2]; }; #if !defined(LUA_IEEEENDIAN) /* { */ #define LUAI_EXTRAIEEE \ static const union luai_Cast ieeeendian = {-(33.0 + 6755399441055744.0)}; -#define LUA_IEEEENDIAN (ieeeendian.l_p[1] == 33) +#define LUA_IEEEENDIANLOC (ieeeendian.l_p[1] == 33) #else +#define LUA_IEEEENDIANLOC LUA_IEEEENDIAN #define LUAI_EXTRAIEEE /* empty */ #endif /* } */ #define lua_number2int32(i,n,t) \ { LUAI_EXTRAIEEE \ volatile union luai_Cast u; u.l_d = (n) + 6755399441055744.0; \ - (i) = (t)u.l_p[LUA_IEEEENDIAN]; } + (i) = (t)u.l_p[LUA_IEEEENDIANLOC]; } #define luai_hashnum(i,n) \ { volatile union luai_Cast u; u.l_d = (n) + 1.0; /* avoid -0 */ \ (i) = u.l_p[0]; (i) += u.l_p[1]; } /* add double bits for his hash */ #define lua_number2int(i,n) lua_number2int32(i, n, int) -#define lua_number2integer(i,n) lua_number2int32(i, n, lua_Integer) #define lua_number2unsigned(i,n) lua_number2int32(i, n, lua_Unsigned) +/* the trick can be expanded to lua_Integer when it is a 32-bit value */ +#if defined(LUA_IEEELL) +#define lua_number2integer(i,n) lua_number2int32(i, n, lua_Integer) +#endif + #endif /* } */ diff --git a/depends/lua/include/lobject.h b/depends/lua/include/lobject.h index 06246bfa5..ca75a028c 100644 --- a/depends/lua/include/lobject.h +++ b/depends/lua/include/lobject.h @@ -1,5 +1,5 @@ /* -** $Id: lobject.h,v 2.64 2011/10/31 17:48:22 roberto Exp $ +** $Id: lobject.h,v 2.70 2012/05/11 14:10:50 roberto Exp $ ** Type definitions for Lua objects ** See Copyright Notice in lua.h */ @@ -36,6 +36,9 @@ ** bit 6: whether value is collectable */ +#define VARBITS (3 << 4) + + /* ** LUA_TFUNCTION variants: ** 0 - Lua function @@ -49,6 +52,12 @@ #define LUA_TCCL (LUA_TFUNCTION | (2 << 4)) /* C closure */ +/* +** LUA_TSTRING variants */ +#define LUA_TSHRSTR (LUA_TSTRING | (0 << 4)) /* short strings */ +#define LUA_TLNGSTR (LUA_TSTRING | (1 << 4)) /* long strings */ + + /* Bit mark for collectable types */ #define BIT_ISCOLLECTABLE (1 << 6) @@ -109,23 +118,28 @@ typedef struct lua_TValue TValue; /* raw type tag of a TValue */ #define rttype(o) ((o)->tt_) +/* tag with no variants (bits 0-3) */ +#define novariant(x) ((x) & 0x0F) + /* type tag of a TValue (bits 0-3 for tags + variant bits 4-5) */ #define ttype(o) (rttype(o) & 0x3F) - /* type tag of a TValue with no variants (bits 0-3) */ -#define ttypenv(o) (rttype(o) & 0x0F) +#define ttypenv(o) (novariant(rttype(o))) /* Macros to test type */ #define checktag(o,t) (rttype(o) == (t)) +#define checktype(o,t) (ttypenv(o) == (t)) #define ttisnumber(o) checktag((o), LUA_TNUMBER) #define ttisnil(o) checktag((o), LUA_TNIL) #define ttisboolean(o) checktag((o), LUA_TBOOLEAN) #define ttislightuserdata(o) checktag((o), LUA_TLIGHTUSERDATA) -#define ttisstring(o) checktag((o), ctb(LUA_TSTRING)) +#define ttisstring(o) checktype((o), LUA_TSTRING) +#define ttisshrstring(o) checktag((o), ctb(LUA_TSHRSTR)) +#define ttislngstring(o) checktag((o), ctb(LUA_TLNGSTR)) #define ttistable(o) checktag((o), ctb(LUA_TTABLE)) -#define ttisfunction(o) (ttypenv(o) == LUA_TFUNCTION) +#define ttisfunction(o) checktype(o, LUA_TFUNCTION) #define ttisclosure(o) ((rttype(o) & 0x1F) == LUA_TFUNCTION) #define ttisCclosure(o) checktag((o), ctb(LUA_TCCL)) #define ttisLclosure(o) checktag((o), ctb(LUA_TLCL)) @@ -161,7 +175,7 @@ typedef struct lua_TValue TValue; /* Macros for internal tests */ -#define righttt(obj) (ttypenv(obj) == gcvalue(obj)->gch.tt) +#define righttt(obj) (ttype(obj) == gcvalue(obj)->gch.tt) #define checkliveness(g,obj) \ lua_longassert(!iscollectable(obj) || \ @@ -193,7 +207,8 @@ typedef struct lua_TValue TValue; #define setsvalue(L,obj,x) \ { TValue *io=(obj); \ - val_(io).gc=cast(GCObject *, (x)); settt_(io, ctb(LUA_TSTRING)); \ + TString *x_ = (x); \ + val_(io).gc=cast(GCObject *, x_); settt_(io, ctb(x_->tsv.tt)); \ checkliveness(G(L),io); } #define setuvalue(L,obj,x) \ @@ -221,11 +236,6 @@ typedef struct lua_TValue TValue; val_(io).gc=cast(GCObject *, (x)); settt_(io, ctb(LUA_TTABLE)); \ checkliveness(G(L),io); } -#define setptvalue(L,obj,x) \ - { TValue *io=(obj); \ - val_(io).gc=cast(GCObject *, (x)); settt_(io, ctb(LUA_TPROTO)); \ - checkliveness(G(L),io); } - #define setdeadvalue(obj) settt_(obj, LUA_TDEADKEY) @@ -256,6 +266,8 @@ typedef struct lua_TValue TValue; #define setsvalue2n setsvalue +/* check whether a number is valid (useful only for NaN trick) */ +#define luai_checknum(L,o,c) { /* empty */ } /* @@ -263,10 +275,7 @@ typedef struct lua_TValue TValue; ** NaN Trick ** ======================================================= */ - -#if defined(LUA_NANTRICK) \ - || defined(LUA_NANTRICK_LE) \ - || defined(LUA_NANTRICK_BE) +#if defined(LUA_NANTRICK) /* ** numbers are represented in the 'd_' field. All other values have the @@ -274,15 +283,23 @@ typedef struct lua_TValue TValue; ** a "signaled NaN", which is never generated by regular operations by ** the CPU (nor by 'strtod') */ -#if !defined(NNMARK) + +/* allows for external implementation for part of the trick */ +#if !defined(NNMARK) /* { */ + + +#if !defined(LUA_IEEEENDIAN) +#error option 'LUA_NANTRICK' needs 'LUA_IEEEENDIAN' +#endif + + #define NNMARK 0x7FF7A500 #define NNMASK 0x7FFFFF00 -#endif #undef TValuefields #undef NILCONSTANT -#if defined(LUA_NANTRICK_LE) +#if (LUA_IEEEENDIAN == 0) /* { */ /* little endian */ #define TValuefields \ @@ -293,7 +310,7 @@ typedef struct lua_TValue TValue; #define d_(o) ((o)->u.d__) #define tt_(o) ((o)->u.i.tt__) -#elif defined(LUA_NANTRICK_BE) +#else /* }{ */ /* big endian */ #define TValuefields \ @@ -304,10 +321,9 @@ typedef struct lua_TValue TValue; #define d_(o) ((o)->u.d__) #define tt_(o) ((o)->u.i.tt__) -#elif !defined(TValuefields) -#error option 'LUA_NANTRICK' needs declaration for 'TValuefields' +#endif /* } */ -#endif +#endif /* } */ /* correspondence with standard representation */ @@ -348,21 +364,18 @@ typedef struct lua_TValue TValue; */ #undef checktag +#undef checktype #define checktag(o,t) (tt_(o) == tag2tt(t)) +#define checktype(o,t) (ctb(tt_(o) | VARBITS) == ctb(tag2tt(t) | VARBITS)) #undef ttisequal #define ttisequal(o1,o2) \ (ttisnumber(o1) ? ttisnumber(o2) : (tt_(o1) == tt_(o2))) - +#undef luai_checknum #define luai_checknum(L,o,c) { if (!ttisnumber(o)) c; } - -#else - -#define luai_checknum(L,o,c) { /* empty */ } - #endif /* }====================================================== */ @@ -401,7 +414,7 @@ typedef union TString { L_Umaxalign dummy; /* ensures maximum alignment for strings */ struct { CommonHeader; - lu_byte reserved; + lu_byte extra; /* reserved words for short strings; "has hash" for longs */ unsigned int hash; size_t len; /* number of characters in string */ } tsv; @@ -501,7 +514,7 @@ typedef struct UpVal { */ #define ClosureHeader \ - CommonHeader; lu_byte isC; lu_byte nupvalues; GCObject *gclist + CommonHeader; lu_byte nupvalues; GCObject *gclist typedef struct CClosure { ClosureHeader; diff --git a/depends/lua/include/lparser.h b/depends/lua/include/lparser.h index caabf46cc..301167d4f 100644 --- a/depends/lua/include/lparser.h +++ b/depends/lua/include/lparser.h @@ -1,5 +1,5 @@ /* -** $Id: lparser.h,v 1.69 2011/07/27 18:09:01 roberto Exp $ +** $Id: lparser.h,v 1.70 2012/05/08 13:53:33 roberto Exp $ ** Lua Parser ** See Copyright Notice in lua.h */ @@ -112,8 +112,8 @@ typedef struct FuncState { } FuncState; -LUAI_FUNC Proto *luaY_parser (lua_State *L, ZIO *z, Mbuffer *buff, - Dyndata *dyd, const char *name, int firstchar); +LUAI_FUNC Closure *luaY_parser (lua_State *L, ZIO *z, Mbuffer *buff, + Dyndata *dyd, const char *name, int firstchar); #endif diff --git a/depends/lua/include/lstate.h b/depends/lua/include/lstate.h index 4743d7410..29f810ba0 100644 --- a/depends/lua/include/lstate.h +++ b/depends/lua/include/lstate.h @@ -1,5 +1,5 @@ /* -** $Id: lstate.h,v 2.74 2011/09/30 12:45:07 roberto Exp $ +** $Id: lstate.h,v 2.81 2012/06/08 15:14:04 roberto Exp $ ** Global State ** See Copyright Notice in lua.h */ @@ -72,6 +72,7 @@ typedef struct CallInfo { struct CallInfo *previous, *next; /* dynamic call link */ short nresults; /* expected number of results from this function */ lu_byte callstatus; + ptrdiff_t extra; union { struct { /* only for Lua functions */ StkId base; /* base for this function */ @@ -81,7 +82,6 @@ typedef struct CallInfo { int ctx; /* context info. in case of yields */ lua_CFunction k; /* continuation in case of yields */ ptrdiff_t old_errfunc; - ptrdiff_t extra; lu_byte old_allowhook; lu_byte status; } c; @@ -100,6 +100,7 @@ typedef struct CallInfo { #define CIST_YPCALL (1<<4) /* call is a yieldable protected call */ #define CIST_STAT (1<<5) /* call has an error status (pcall) */ #define CIST_TAIL (1<<6) /* call was tail called */ +#define CIST_HOOKYIELD (1<<7) /* last hook called yielded */ #define isLua(ci) ((ci)->callstatus & CIST_LUA) @@ -113,9 +114,11 @@ typedef struct global_State { void *ud; /* auxiliary data to `frealloc' */ lu_mem totalbytes; /* number of bytes currently allocated - GCdebt */ l_mem GCdebt; /* bytes allocated not yet compensated by the collector */ - lu_mem lastmajormem; /* memory in use after last major collection */ + lu_mem GCmemtrav; /* memory traversed by the GC */ + lu_mem GCestimate; /* an estimate of the non-garbage memory in use */ stringtable strt; /* hash table for strings */ TValue l_registry; + unsigned int seed; /* randomized seed for hashes */ lu_byte currentwhite; lu_byte gcstate; /* state of garbage collector */ lu_byte gckind; /* kind of GC running */ @@ -123,7 +126,8 @@ typedef struct global_State { int sweepstrgc; /* position of sweep in `strt' */ GCObject *allgc; /* list of all collectable objects */ GCObject *finobj; /* list of collectable objects with finalizers */ - GCObject **sweepgc; /* current position of sweep */ + GCObject **sweepgc; /* current position of sweep in list 'allgc' */ + GCObject **sweepfin; /* current position of sweep in list 'finobj' */ GCObject *gray; /* list of gray objects */ GCObject *grayagain; /* list of objects to be traversed atomically */ GCObject *weak; /* list of tables with weak values */ @@ -193,11 +197,15 @@ union GCObject { #define gch(o) (&(o)->gch) /* macros to convert a GCObject into a specific value */ -#define rawgco2ts(o) check_exp((o)->gch.tt == LUA_TSTRING, &((o)->ts)) +#define rawgco2ts(o) \ + check_exp(novariant((o)->gch.tt) == LUA_TSTRING, &((o)->ts)) #define gco2ts(o) (&rawgco2ts(o)->tsv) #define rawgco2u(o) check_exp((o)->gch.tt == LUA_TUSERDATA, &((o)->u)) #define gco2u(o) (&rawgco2u(o)->uv) -#define gco2cl(o) check_exp((o)->gch.tt == LUA_TFUNCTION, &((o)->cl)) +#define gco2lcl(o) check_exp((o)->gch.tt == LUA_TLCL, &((o)->cl.l)) +#define gco2ccl(o) check_exp((o)->gch.tt == LUA_TCCL, &((o)->cl.c)) +#define gco2cl(o) \ + check_exp(novariant((o)->gch.tt) == LUA_TFUNCTION, &((o)->cl)) #define gco2t(o) check_exp((o)->gch.tt == LUA_TTABLE, &((o)->h)) #define gco2p(o) check_exp((o)->gch.tt == LUA_TPROTO, &((o)->p)) #define gco2uv(o) check_exp((o)->gch.tt == LUA_TUPVAL, &((o)->uv)) diff --git a/depends/lua/include/lstring.h b/depends/lua/include/lstring.h index d708a1b09..d312ff3d2 100644 --- a/depends/lua/include/lstring.h +++ b/depends/lua/include/lstring.h @@ -1,5 +1,5 @@ /* -** $Id: lstring.h,v 1.46 2010/04/05 16:26:37 roberto Exp $ +** $Id: lstring.h,v 1.49 2012/02/01 21:57:15 roberto Exp $ ** String table (keep all strings handled by Lua) ** See Copyright Notice in lua.h */ @@ -23,11 +23,20 @@ /* -** as all string are internalized, string equality becomes -** pointer equality +** test whether a string is a reserved word */ -#define eqstr(a,b) ((a) == (b)) +#define isreserved(s) ((s)->tsv.tt == LUA_TSHRSTR && (s)->tsv.extra > 0) + +/* +** equality for short strings, which are always internalized +*/ +#define eqshrstr(a,b) check_exp((a)->tsv.tt == LUA_TSHRSTR, (a) == (b)) + + +LUAI_FUNC unsigned int luaS_hash (const char *str, size_t l, unsigned int seed); +LUAI_FUNC int luaS_eqlngstr (TString *a, TString *b); +LUAI_FUNC int luaS_eqstr (TString *a, TString *b); LUAI_FUNC void luaS_resize (lua_State *L, int newsize); LUAI_FUNC Udata *luaS_newudata (lua_State *L, size_t s, Table *e); LUAI_FUNC TString *luaS_newlstr (lua_State *L, const char *str, size_t l); diff --git a/depends/lua/include/lua.h b/depends/lua/include/lua.h index 1fafa45e4..a3a3a70c6 100644 --- a/depends/lua/include/lua.h +++ b/depends/lua/include/lua.h @@ -1,5 +1,5 @@ /* -** $Id: lua.h,v 1.282 2011/11/29 15:55:08 roberto Exp $ +** $Id: lua.h,v 1.283 2012/04/20 13:18:26 roberto Exp $ ** Lua - A Scripting Language ** Lua.org, PUC-Rio, Brazil (http://www.lua.org) ** See Copyright Notice at the end of this file @@ -19,11 +19,11 @@ #define LUA_VERSION_MAJOR "5" #define LUA_VERSION_MINOR "2" #define LUA_VERSION_NUM 502 -#define LUA_VERSION_RELEASE "0" +#define LUA_VERSION_RELEASE "1" #define LUA_VERSION "Lua " LUA_VERSION_MAJOR "." LUA_VERSION_MINOR #define LUA_RELEASE LUA_VERSION "." LUA_VERSION_RELEASE -#define LUA_COPYRIGHT LUA_RELEASE " Copyright (C) 1994-2011 Lua.org, PUC-Rio" +#define LUA_COPYRIGHT LUA_RELEASE " Copyright (C) 1994-2012 Lua.org, PUC-Rio" #define LUA_AUTHORS "R. Ierusalimschy, L. H. de Figueiredo, W. Celes" @@ -413,7 +413,7 @@ struct lua_Debug { /****************************************************************************** -* Copyright (C) 1994-2011 Lua.org, PUC-Rio. All rights reserved. +* Copyright (C) 1994-2012 Lua.org, PUC-Rio. * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the diff --git a/depends/lua/include/luaconf.h b/depends/lua/include/luaconf.h index fca66ce48..e4335df90 100644 --- a/depends/lua/include/luaconf.h +++ b/depends/lua/include/luaconf.h @@ -1,5 +1,5 @@ /* -** $Id: luaconf.h,v 1.170 2011/12/06 16:58:36 roberto Exp $ +** $Id: luaconf.h,v 1.172 2012/05/11 14:14:42 roberto Exp $ ** Configuration file for Lua ** See Copyright Notice in lua.h */ @@ -89,21 +89,25 @@ ** In Windows, any exclamation mark ('!') in the path is replaced by the ** path of the directory of the executable file of the current process. */ -#define LUA_LDIR "!\\hack\\lua\\" +#define LUA_LDIR "!\\lua\\" #define LUA_CDIR "!\\" #define LUA_PATH_DEFAULT \ - LUA_LDIR"?.lua;" LUA_LDIR"?\\init.lua;" ".\\?.lua" + LUA_LDIR"?.lua;" LUA_LDIR"?\\init.lua;" \ + LUA_CDIR"?.lua;" LUA_CDIR"?\\init.lua;" ".\\?.lua" #define LUA_CPATH_DEFAULT \ - LUA_CDIR"?.dll;" ".\\?.dll" + LUA_CDIR"?.dll;" LUA_CDIR"loadall.dll;" ".\\?.dll" #else /* }{ */ -#define LUA_LDIR "./hack/lua/" -#define LUA_CDIR "./hack/" +#define LUA_VDIR LUA_VERSION_MAJOR "." LUA_VERSION_MINOR "/" +#define LUA_ROOT "/usr/local/" +#define LUA_LDIR LUA_ROOT "share/lua/" LUA_VDIR +#define LUA_CDIR LUA_ROOT "lib/lua/" LUA_VDIR #define LUA_PATH_DEFAULT \ - LUA_LDIR"?.lua;" LUA_LDIR"?/init.lua;" "./?.lua" + LUA_LDIR"?.lua;" LUA_LDIR"?/init.lua;" \ + LUA_CDIR"?.lua;" LUA_CDIR"?/init.lua;" "./?.lua" #define LUA_CPATH_DEFAULT \ - LUA_CDIR"?.so;" "./?.so" + LUA_CDIR"?.so;" LUA_CDIR"loadall.so;" "./?.so" #endif /* } */ @@ -138,15 +142,11 @@ */ #if defined(LUA_BUILD_AS_DLL) /* { */ -#if defined(_MSC_VER) #if defined(LUA_CORE) || defined(LUA_LIB) /* { */ #define LUA_API __declspec(dllexport) #else /* }{ */ #define LUA_API __declspec(dllimport) #endif /* } */ -#else -#define LUA_API __attribute__ ((visibility("default"))) -#endif #else /* }{ */ @@ -223,6 +223,13 @@ (fprintf(stderr, (s), (p)), fflush(stderr)) +/* +@@ LUAI_MAXSHORTLEN is the maximum length for short strings, that is, +** strings that are internalized. (Cannot be smaller than reserved words +** or tags for metamethods, as these strings must be internalized; +** #("function") = 8, #("__newindex") = 10.) +*/ +#define LUAI_MAXSHORTLEN 40 @@ -453,66 +460,76 @@ #define LUA_UNSIGNED unsigned LUA_INT32 -#if defined(LUA_CORE) /* { */ -#if defined(LUA_NUMBER_DOUBLE) && !defined(LUA_ANSI) /* { */ +/* +** Some tricks with doubles +*/ + +#if defined(LUA_CORE) && \ + defined(LUA_NUMBER_DOUBLE) && !defined(LUA_ANSI) /* { */ +/* +** The next definitions activate some tricks to speed up the +** conversion from doubles to integer types, mainly to LUA_UNSIGNED. +** +@@ MS_ASMTRICK uses Microsoft assembler to avoid clashes with a +** DirectX idiosyncrasy. +** +@@ LUA_IEEE754TRICK uses a trick that should work on any machine +** using IEEE754 with a 32-bit integer type. +** +@@ LUA_IEEELL extends the trick to LUA_INTEGER; should only be +** defined when LUA_INTEGER is a 32-bit integer. +** +@@ LUA_IEEEENDIAN is the endianness of doubles in your machine +** (0 for little endian, 1 for big endian); if not defined, Lua will +** check it dynamically for LUA_IEEE754TRICK (but not for LUA_NANTRICK). +** +@@ LUA_NANTRICK controls the use of a trick to pack all types into +** a single double value, using NaN values to represent non-number +** values. The trick only works on 32-bit machines (ints and pointers +** are 32-bit values) with numbers represented as IEEE 754-2008 doubles +** with conventional endianess (12345678 or 87654321), in CPUs that do +** not produce signaling NaN values (all NaNs are quiet). +*/ -/* On a Microsoft compiler on a Pentium, use assembler to avoid clashes - with a DirectX idiosyncrasy */ +/* Microsoft compiler on a Pentium (32 bit) ? */ #if defined(LUA_WIN) && defined(_MSC_VER) && defined(_M_IX86) /* { */ #define MS_ASMTRICK +#define LUA_IEEEENDIAN 0 +#define LUA_NANTRICK -#else /* }{ */ -/* the next definition uses a trick that should work on any machine - using IEEE754 with a 32-bit integer type */ - -#define LUA_IEEE754TRICK - -/* -@@ LUA_IEEEENDIAN is the endianness of doubles in your machine -** (0 for little endian, 1 for big endian); if not defined, Lua will -** check it dynamically. -*/ -/* check for known architectures */ -#if defined(__i386__) || defined(__i386) || defined(__X86__) || \ - defined (__x86_64) -#define LUA_IEEEENDIAN 0 -#elif defined(__POWERPC__) || defined(__ppc__) -#define LUA_IEEEENDIAN 1 -#endif -#endif /* } */ +/* pentium 32 bits? */ +#elif defined(__i386__) || defined(__i386) || defined(__X86__) /* }{ */ -#endif /* } */ +#define LUA_IEEE754TRICK +#define LUA_IEEELL +#define LUA_IEEEENDIAN 0 +#define LUA_NANTRICK -#endif /* } */ +/* pentium 64 bits? */ +#elif defined(__x86_64) /* }{ */ -/* }================================================================== */ +#define LUA_IEEE754TRICK +#define LUA_IEEEENDIAN 0 +#elif defined(__POWERPC__) || defined(__ppc__) /* }{ */ -/* -@@ LUA_NANTRICK_LE/LUA_NANTRICK_BE controls the use of a trick to -** pack all types into a single double value, using NaN values to -** represent non-number values. The trick only works on 32-bit machines -** (ints and pointers are 32-bit values) with numbers represented as -** IEEE 754-2008 doubles with conventional endianess (12345678 or -** 87654321), in CPUs that do not produce signaling NaN values (all NaNs -** are quiet). -*/ -#if defined(LUA_CORE) && \ - defined(LUA_NUMBER_DOUBLE) && !defined(LUA_ANSI) /* { */ +#define LUA_IEEE754TRICK +#define LUA_IEEEENDIAN 1 -/* little-endian architectures that satisfy those conditions */ -#if defined(__i386__) || defined(__i386) || defined(__X86__) || \ - defined(_M_IX86) +#else /* }{ */ -#define LUA_NANTRICK_LE +/* assume IEEE754 and a 32-bit integer type */ +#define LUA_IEEE754TRICK -#endif +#endif /* } */ #endif /* } */ +/* }================================================================== */ + diff --git a/depends/lua/include/lundump.h b/depends/lua/include/lundump.h index b63993ffe..2b8accecb 100644 --- a/depends/lua/include/lundump.h +++ b/depends/lua/include/lundump.h @@ -1,5 +1,5 @@ /* -** $Id: lundump.h,v 1.44 2011/05/06 13:35:17 lhf Exp $ +** $Id: lundump.h,v 1.39 2012/05/08 13:53:33 roberto Exp $ ** load precompiled Lua chunks ** See Copyright Notice in lua.h */ @@ -11,7 +11,7 @@ #include "lzio.h" /* load one chunk; from lundump.c */ -LUAI_FUNC Proto* luaU_undump (lua_State* L, ZIO* Z, Mbuffer* buff, const char* name); +LUAI_FUNC Closure* luaU_undump (lua_State* L, ZIO* Z, Mbuffer* buff, const char* name); /* make header; from lundump.c */ LUAI_FUNC void luaU_header (lu_byte* h); diff --git a/depends/lua/src/lapi.c b/depends/lua/src/lapi.c index e96ceb9e1..1854fe615 100644 --- a/depends/lua/src/lapi.c +++ b/depends/lua/src/lapi.c @@ -1,5 +1,5 @@ /* -** $Id: lapi.c,v 2.159 2011/11/30 12:32:05 roberto Exp $ +** $Id: lapi.c,v 2.164 2012/06/08 15:14:04 roberto Exp $ ** Lua API ** See Copyright Notice in lua.h */ @@ -950,7 +950,7 @@ LUA_API int lua_pcallk (lua_State *L, int nargs, int nresults, int errfunc, ci->u.c.k = k; /* save continuation */ ci->u.c.ctx = ctx; /* save context */ /* save information for error recovery */ - ci->u.c.extra = savestack(L, c.func); + ci->extra = savestack(L, c.func); ci->u.c.old_allowhook = L->allowhook; ci->u.c.old_errfunc = L->errfunc; L->errfunc = func; @@ -1045,17 +1045,17 @@ LUA_API int lua_gc (lua_State *L, int what, int data) { } case LUA_GCSTEP: { if (g->gckind == KGC_GEN) { /* generational mode? */ - res = (g->lastmajormem == 0); /* 1 if will do major collection */ + res = (g->GCestimate == 0); /* true if it will do major collection */ luaC_forcestep(L); /* do a single step */ } else { - while (data-- >= 0) { - luaC_forcestep(L); - if (g->gcstate == GCSpause) { /* end of cycle? */ - res = 1; /* signal it */ - break; - } - } + lu_mem debt = cast(lu_mem, data) * 1024 - GCSTEPSIZE; + if (g->gcrunning) + debt += g->GCdebt; /* include current debt */ + luaE_setdebt(g, debt); + luaC_forcestep(L); + if (g->gcstate == GCSpause) /* end of cycle? */ + res = 1; /* signal it */ } break; } diff --git a/depends/lua/src/lauxlib.c b/depends/lua/src/lauxlib.c index c1b715f34..36ae7e629 100644 --- a/depends/lua/src/lauxlib.c +++ b/depends/lua/src/lauxlib.c @@ -1,5 +1,5 @@ /* -** $Id: lauxlib.c,v 1.240 2011/12/06 16:33:55 roberto Exp $ +** $Id: lauxlib.c,v 1.244 2012/05/31 20:28:45 roberto Exp $ ** Auxiliary functions for building Lua libraries ** See Copyright Notice in lua.h */ @@ -520,11 +520,11 @@ LUALIB_API char *luaL_buffinitsize (lua_State *L, luaL_Buffer *B, size_t sz) { LUALIB_API int luaL_ref (lua_State *L, int t) { int ref; - t = lua_absindex(L, t); if (lua_isnil(L, -1)) { lua_pop(L, 1); /* remove from stack */ return LUA_REFNIL; /* `nil' has a unique fixed reference */ } + t = lua_absindex(L, t); lua_rawgeti(L, t, freelist); /* get first free element */ ref = (int)lua_tointeger(L, -1); /* ref = t[freelist] */ lua_pop(L, 1); /* remove it from stack */ @@ -616,8 +616,10 @@ static int skipBOM (LoadF *lf) { static int skipcomment (LoadF *lf, int *cp) { int c = *cp = skipBOM(lf); if (c == '#') { /* first line is a comment (Unix exec. file)? */ - while ((c = getc(lf->f)) != EOF && c != '\n') ; /* skip first line */ - *cp = getc(lf->f); /* skip end-of-line */ + do { /* skip first line */ + c = getc(lf->f); + } while (c != EOF && c != '\n') ; + *cp = getc(lf->f); /* skip end-of-line, if present */ return 1; /* there was a comment */ } else return 0; /* no comment */ @@ -843,6 +845,7 @@ LUALIB_API void luaL_openlib (lua_State *L, const char *libname, ** Returns with only the table at the stack. */ LUALIB_API void luaL_setfuncs (lua_State *L, const luaL_Reg *l, int nup) { + luaL_checkversion(L); luaL_checkstack(L, nup, "too many upvalues"); for (; l->name != NULL; l++) { /* fill the table with given functions */ int i; @@ -889,10 +892,8 @@ LUALIB_API void luaL_requiref (lua_State *L, const char *modname, lua_setfield(L, -2, modname); /* _LOADED[modname] = module */ lua_pop(L, 1); /* remove _LOADED table */ if (glb) { - lua_pushglobaltable(L); - lua_pushvalue(L, -2); /* copy of 'mod' */ - lua_setfield(L, -2, modname); /* _G[modname] = module */ - lua_pop(L, 1); /* remove _G table */ + lua_pushvalue(L, -1); /* copy of 'mod' */ + lua_setglobal(L, modname); /* _G[modname] = module */ } } diff --git a/depends/lua/src/lbaselib.c b/depends/lua/src/lbaselib.c index 1dfae3014..dbfcb02cf 100644 --- a/depends/lua/src/lbaselib.c +++ b/depends/lua/src/lbaselib.c @@ -1,5 +1,5 @@ /* -** $Id: lbaselib.c,v 1.273 2011/11/30 13:03:24 roberto Exp $ +** $Id: lbaselib.c,v 1.274 2012/04/27 14:13:19 roberto Exp $ ** Basic library ** See Copyright Notice in lua.h */ @@ -293,6 +293,7 @@ static const char *generic_reader (lua_State *L, void *ud, size_t *size) { lua_pushvalue(L, 1); /* get function */ lua_call(L, 0, 1); /* call it */ if (lua_isnil(L, -1)) { + lua_pop(L, 1); /* pop result */ *size = 0; return NULL; } diff --git a/depends/lua/src/lcorolib.c b/depends/lua/src/lcorolib.c index 0edde26da..c7932d90f 100644 --- a/depends/lua/src/lcorolib.c +++ b/depends/lua/src/lcorolib.c @@ -1,5 +1,5 @@ /* -** $Id: lcorolib.c,v 1.3 2011/08/23 17:24:34 roberto Exp $ +** $Id: lcorolib.c,v 1.4 2012/04/27 18:59:04 roberto Exp $ ** Coroutine Library ** See Copyright Notice in lua.h */ @@ -80,8 +80,9 @@ static int luaB_auxwrap (lua_State *L) { static int luaB_cocreate (lua_State *L) { - lua_State *NL = lua_newthread(L); + lua_State *NL; luaL_checktype(L, 1, LUA_TFUNCTION); + NL = lua_newthread(L); lua_pushvalue(L, 1); /* move function to top */ lua_xmove(L, NL, 1); /* move function from L to NL */ return 1; diff --git a/depends/lua/src/ldblib.c b/depends/lua/src/ldblib.c index 3c2f15957..c02269457 100644 --- a/depends/lua/src/ldblib.c +++ b/depends/lua/src/ldblib.c @@ -1,5 +1,5 @@ /* -** $Id: ldblib.c,v 1.131 2011/10/24 14:54:05 roberto Exp $ +** $Id: ldblib.c,v 1.132 2012/01/19 20:14:44 roberto Exp $ ** Interface from Lua to its debug API ** See Copyright Notice in lua.h */ @@ -253,14 +253,15 @@ static int db_upvaluejoin (lua_State *L) { } -#define gethooktable(L) luaL_getsubtable(L, LUA_REGISTRYINDEX, HOOKKEY); +#define gethooktable(L) luaL_getsubtable(L, LUA_REGISTRYINDEX, HOOKKEY) static void hookf (lua_State *L, lua_Debug *ar) { static const char *const hooknames[] = {"call", "return", "line", "count", "tail call"}; gethooktable(L); - lua_rawgetp(L, -1, L); + lua_pushthread(L); + lua_rawget(L, -2); if (lua_isfunction(L, -1)) { lua_pushstring(L, hooknames[(int)ar->event]); if (ar->currentline >= 0) @@ -306,10 +307,15 @@ static int db_sethook (lua_State *L) { count = luaL_optint(L, arg+3, 0); func = hookf; mask = makemask(smask, count); } - gethooktable(L); + if (gethooktable(L) == 0) { /* creating hook table? */ + lua_pushstring(L, "k"); + lua_setfield(L, -2, "__mode"); /** hooktable.__mode = "k" */ + lua_pushvalue(L, -1); + lua_setmetatable(L, -2); /* setmetatable(hooktable) = hooktable */ + } + lua_pushthread(L1); lua_xmove(L1, L, 1); lua_pushvalue(L, arg+1); - lua_rawsetp(L, -2, L1); /* set new hook */ - lua_pop(L, 1); /* remove hook table */ + lua_rawset(L, -3); /* set new hook */ lua_sethook(L1, func, mask, count); /* set hooks */ return 0; } @@ -325,7 +331,8 @@ static int db_gethook (lua_State *L) { lua_pushliteral(L, "external hook"); else { gethooktable(L); - lua_rawgetp(L, -1, L1); /* get hook */ + lua_pushthread(L1); lua_xmove(L1, L, 1); + lua_rawget(L, -2); /* get hook */ lua_remove(L, -2); /* remove hook table */ } lua_pushstring(L, unmakemask(mask, buff)); diff --git a/depends/lua/src/ldebug.c b/depends/lua/src/ldebug.c index 31b7ae401..43f8f046f 100644 --- a/depends/lua/src/ldebug.c +++ b/depends/lua/src/ldebug.c @@ -1,5 +1,5 @@ /* -** $Id: ldebug.c,v 2.88 2011/11/30 12:43:51 roberto Exp $ +** $Id: ldebug.c,v 2.89 2012/01/20 22:05:50 roberto Exp $ ** Debug Interface ** See Copyright Notice in lua.h */ @@ -30,6 +30,9 @@ +#define noLuaClosure(f) ((f) == NULL || (f)->c.tt == LUA_TCCL) + + static const char *getfuncname (lua_State *L, CallInfo *ci, const char **name); @@ -173,7 +176,7 @@ LUA_API const char *lua_setlocal (lua_State *L, const lua_Debug *ar, int n) { static void funcinfo (lua_Debug *ar, Closure *cl) { - if (cl == NULL || cl->c.isC) { + if (noLuaClosure(cl)) { ar->source = "=[C]"; ar->linedefined = -1; ar->lastlinedefined = -1; @@ -191,7 +194,7 @@ static void funcinfo (lua_Debug *ar, Closure *cl) { static void collectvalidlines (lua_State *L, Closure *f) { - if (f == NULL || f->c.isC) { + if (noLuaClosure(f)) { setnilvalue(L->top); incr_top(L); } @@ -210,7 +213,7 @@ static void collectvalidlines (lua_State *L, Closure *f) { static int auxgetinfo (lua_State *L, const char *what, lua_Debug *ar, - Closure *f, CallInfo *ci) { + Closure *f, CallInfo *ci) { int status = 1; for (; *what; what++) { switch (*what) { @@ -224,7 +227,7 @@ static int auxgetinfo (lua_State *L, const char *what, lua_Debug *ar, } case 'u': { ar->nups = (f == NULL) ? 0 : f->c.nupvalues; - if (f == NULL || f->c.isC) { + if (noLuaClosure(f)) { ar->isvararg = 1; ar->nparams = 0; } diff --git a/depends/lua/src/ldo.c b/depends/lua/src/ldo.c index e0f13c44e..d18e33cd4 100644 --- a/depends/lua/src/ldo.c +++ b/depends/lua/src/ldo.c @@ -1,5 +1,5 @@ /* -** $Id: ldo.c,v 2.102 2011/11/29 15:55:08 roberto Exp $ +** $Id: ldo.c,v 2.105 2012/06/08 15:14:04 roberto Exp $ ** Stack and Call structure of Lua ** See Copyright Notice in lua.h */ @@ -402,8 +402,6 @@ static void finishCcall (lua_State *L) { int n; lua_assert(ci->u.c.k != NULL); /* must have a continuation */ lua_assert(L->nny == 0); - /* finish 'luaD_call' */ - //L->nCcalls--; /* finish 'lua_callk' */ adjustresults(L, ci->nresults); /* call continuation function */ @@ -453,7 +451,7 @@ static int recover (lua_State *L, int status) { CallInfo *ci = findpcall(L); if (ci == NULL) return 0; /* no recovery point */ /* "finish" luaD_pcall */ - oldtop = restorestack(L, ci->u.c.extra); + oldtop = restorestack(L, ci->extra); luaF_close(L, oldtop); seterrorobj(L, status, oldtop); L->ci = ci; @@ -484,9 +482,10 @@ static l_noret resume_error (lua_State *L, const char *msg, StkId firstArg) { ** do the work for 'lua_resume' in protected mode */ static void resume (lua_State *L, void *ud) { + int nCcalls = L->nCcalls; StkId firstArg = cast(StkId, ud); CallInfo *ci = L->ci; - if (L->nCcalls >= LUAI_MAXCCALLS) + if (nCcalls >= LUAI_MAXCCALLS) resume_error(L, "C stack overflow", firstArg); if (L->status == LUA_OK) { /* may be starting a coroutine */ if (ci != &L->base_ci) /* not in base level? */ @@ -499,10 +498,10 @@ static void resume (lua_State *L, void *ud) { resume_error(L, "cannot resume dead coroutine", firstArg); else { /* resuming from previous yield */ L->status = LUA_OK; + ci->func = restorestack(L, ci->extra); if (isLua(ci)) /* yielded inside a hook? */ luaV_execute(L); /* just continue running Lua code */ else { /* 'common' yield */ - ci->func = restorestack(L, ci->u.c.extra); if (ci->u.c.k != NULL) { /* does it have a continuation? */ int n; ci->u.c.status = LUA_YIELD; /* 'default' status */ @@ -513,11 +512,11 @@ static void resume (lua_State *L, void *ud) { api_checknelems(L, n); firstArg = L->top - n; /* yield results come from continuation */ } - //L->nCcalls--; /* finish 'luaD_call' */ luaD_poscall(L, firstArg); /* finish 'luaD_precall' */ } unroll(L, NULL); } + lua_assert(nCcalls == L->nCcalls); } @@ -564,13 +563,13 @@ LUA_API int lua_yieldk (lua_State *L, int nresults, int ctx, lua_CFunction k) { luaG_runerror(L, "attempt to yield from outside a coroutine"); } L->status = LUA_YIELD; + ci->extra = savestack(L, ci->func); /* save current 'func' */ if (isLua(ci)) { /* inside a hook? */ api_check(L, k == NULL, "hooks cannot continue after yielding"); } else { if ((ci->u.c.k = k) != NULL) /* is there a continuation? */ ci->u.c.ctx = ctx; /* save context */ - ci->u.c.extra = savestack(L, ci->func); /* save current 'func' */ ci->func = L->top - nresults - 1; /* protect stack below results */ luaD_throw(L, LUA_YIELD); } @@ -627,24 +626,23 @@ static void checkmode (lua_State *L, const char *mode, const char *x) { static void f_parser (lua_State *L, void *ud) { int i; - Proto *tf; Closure *cl; struct SParser *p = cast(struct SParser *, ud); int c = zgetc(p->z); /* read first character */ if (c == LUA_SIGNATURE[0]) { checkmode(L, p->mode, "binary"); - tf = luaU_undump(L, p->z, &p->buff, p->name); + cl = luaU_undump(L, p->z, &p->buff, p->name); } else { checkmode(L, p->mode, "text"); - tf = luaY_parser(L, p->z, &p->buff, &p->dyd, p->name, c); + cl = luaY_parser(L, p->z, &p->buff, &p->dyd, p->name, c); + } + lua_assert(cl->l.nupvalues == cl->l.p->sizeupvalues); + for (i = 0; i < cl->l.nupvalues; i++) { /* initialize upvalues */ + UpVal *up = luaF_newupval(L); + cl->l.upvals[i] = up; + luaC_objbarrier(L, cl, up); } - setptvalue2s(L, L->top, tf); - incr_top(L); - cl = luaF_newLclosure(L, tf); - setclLvalue(L, L->top - 1, cl); - for (i = 0; i < tf->sizeupvalues; i++) /* initialize upvalues */ - cl->l.upvals[i] = luaF_newupval(L); } diff --git a/depends/lua/src/ldump.c b/depends/lua/src/ldump.c index 699e1dc4e..d5e6a47cb 100644 --- a/depends/lua/src/ldump.c +++ b/depends/lua/src/ldump.c @@ -1,5 +1,5 @@ /* -** $Id: ldump.c,v 1.19 2011/11/23 17:48:18 lhf Exp $ +** $Id: ldump.c,v 2.17 2012/01/23 23:02:10 roberto Exp $ ** save precompiled Lua chunks ** See Copyright Notice in lua.h */ @@ -84,8 +84,8 @@ static void DumpConstants(const Proto* f, DumpState* D) for (i=0; ik[i]; - DumpChar(ttype(o),D); - switch (ttype(o)) + DumpChar(ttypenv(o),D); + switch (ttypenv(o)) { case LUA_TNIL: break; @@ -98,6 +98,7 @@ static void DumpConstants(const Proto* f, DumpState* D) case LUA_TSTRING: DumpString(rawtsvalue(o),D); break; + default: lua_assert(0); } } n=f->sizep; diff --git a/depends/lua/src/lfunc.c b/depends/lua/src/lfunc.c index 1a1a8bb8b..4fd27fe5e 100644 --- a/depends/lua/src/lfunc.c +++ b/depends/lua/src/lfunc.c @@ -1,5 +1,5 @@ /* -** $Id: lfunc.c,v 2.27 2010/06/30 14:11:17 roberto Exp $ +** $Id: lfunc.c,v 2.29 2012/05/08 13:53:33 roberto Exp $ ** Auxiliary functions to manipulate prototypes and closures ** See Copyright Notice in lua.h */ @@ -21,18 +21,15 @@ Closure *luaF_newCclosure (lua_State *L, int n) { - Closure *c = &luaC_newobj(L, LUA_TFUNCTION, sizeCclosure(n), NULL, 0)->cl; - c->c.isC = 1; + Closure *c = &luaC_newobj(L, LUA_TCCL, sizeCclosure(n), NULL, 0)->cl; c->c.nupvalues = cast_byte(n); return c; } -Closure *luaF_newLclosure (lua_State *L, Proto *p) { - int n = p->sizeupvalues; - Closure *c = &luaC_newobj(L, LUA_TFUNCTION, sizeLclosure(n), NULL, 0)->cl; - c->l.isC = 0; - c->l.p = p; +Closure *luaF_newLclosure (lua_State *L, int n) { + Closure *c = &luaC_newobj(L, LUA_TLCL, sizeLclosure(n), NULL, 0)->cl; + c->l.p = NULL; c->l.nupvalues = cast_byte(n); while (n--) c->l.upvals[n] = NULL; return c; @@ -146,13 +143,6 @@ void luaF_freeproto (lua_State *L, Proto *f) { } -void luaF_freeclosure (lua_State *L, Closure *c) { - int size = (c->c.isC) ? sizeCclosure(c->c.nupvalues) : - sizeLclosure(c->l.nupvalues); - luaM_freemem(L, c, size); -} - - /* ** Look for n-th local variable at line `line' in function `func'. ** Returns NULL if not found. diff --git a/depends/lua/src/lgc.c b/depends/lua/src/lgc.c index cdd92e524..06f972a72 100644 --- a/depends/lua/src/lgc.c +++ b/depends/lua/src/lgc.c @@ -1,5 +1,5 @@ /* -** $Id: lgc.c,v 2.116 2011/12/02 13:18:41 roberto Exp $ +** $Id: lgc.c,v 2.133 2012/05/31 21:28:59 roberto Exp $ ** Garbage Collector ** See Copyright Notice in lua.h */ @@ -24,34 +24,40 @@ -/* how much to allocate before next GC step */ -#define GCSTEPSIZE 1024 +/* +** cost of sweeping one element (the size of a small object divided +** by some adjust for the sweep speed) +*/ +#define GCSWEEPCOST ((sizeof(TString) + 4) / 4) /* maximum number of elements to sweep in each single step */ -#define GCSWEEPMAX 40 - -/* cost of sweeping one element */ -#define GCSWEEPCOST 1 +#define GCSWEEPMAX (cast_int((GCSTEPSIZE / GCSWEEPCOST) / 4)) /* maximum number of finalizers to call in each GC step */ #define GCFINALIZENUM 4 -/* cost of marking the root set */ -#define GCROOTCOST 10 -/* cost of atomic step */ -#define GCATOMICCOST 1000 +/* +** macro to adjust 'stepmul': 'stepmul' is actually used like +** 'stepmul / STEPMULADJ' (value chosen by tests) +*/ +#define STEPMULADJ 200 + +/* +** macro to adjust 'pause': 'pause' is actually used like +** 'pause / PAUSEADJ' (value chosen by tests) +*/ +#define PAUSEADJ 200 + -/* basic cost to traverse one object (to be added to the links the - object may have) */ -#define TRAVCOST 5 /* ** standard negative debt for GC; a reasonable "time" to wait before ** starting a new cycle */ -#define stddebt(g) (-cast(l_mem, gettotalbytes(g)/100) * g->gcpause) +#define stddebtest(g,e) (-cast(l_mem, (e)/PAUSEADJ) * g->gcpause) +#define stddebt(g) stddebtest(g, gettotalbytes(g)) /* @@ -65,8 +71,6 @@ #define white2gray(x) resetbits(gch(x)->marked, WHITEBITS) #define black2gray(x) resetbit(gch(x)->marked, BLACKBIT) -#define stringmark(s) ((void)((s) && resetbits((s)->tsv.marked, WHITEBITS))) - #define isfinalized(x) testbit(gch(x)->marked, FINALIZEDBIT) @@ -123,10 +127,10 @@ static void removeentry (Node *n) { ** other objects: if really collected, cannot keep them; for objects ** being finalized, keep them in keys, but not in values */ -static int iscleared (const TValue *o) { +static int iscleared (global_State *g, const TValue *o) { if (!iscollectable(o)) return 0; else if (ttisstring(o)) { - stringmark(rawtsvalue(o)); /* strings are `values', so are never weak */ + markobject(g, rawtsvalue(o)); /* strings are `values', so are never weak */ return 0; } else return iswhite(gcvalue(o)); @@ -217,7 +221,8 @@ void luaC_checkupvalcolor (global_State *g, UpVal *uv) { GCObject *luaC_newobj (lua_State *L, int tt, size_t sz, GCObject **list, int offset) { global_State *g = G(L); - GCObject *o = obj2gco(cast(char *, luaM_newobject(L, tt, sz)) + offset); + char *raw = cast(char *, luaM_newobject(L, novariant(tt), sz)); + GCObject *o = obj2gco(raw + offset); if (list == NULL) list = &g->allgc; /* standard list for collectable objects */ gch(o)->marked = luaC_white(g); @@ -239,54 +244,63 @@ GCObject *luaC_newobj (lua_State *L, int tt, size_t sz, GCObject **list, /* -** mark an object. Userdata and closed upvalues are visited and turned -** black here. Strings remain gray (it is the same as making them -** black). Other objects are marked gray and added to appropriate list -** to be visited (and turned black) later. (Open upvalues are already -** linked in 'headuv' list.) +** mark an object. Userdata, strings, and closed upvalues are visited +** and turned black here. Other objects are marked gray and added +** to appropriate list to be visited (and turned black) later. (Open +** upvalues are already linked in 'headuv' list.) */ static void reallymarkobject (global_State *g, GCObject *o) { - lua_assert(iswhite(o) && !isdead(g, o)); + lu_mem size; white2gray(o); switch (gch(o)->tt) { - case LUA_TSTRING: { - return; /* for strings, gray is as good as black */ + case LUA_TSHRSTR: + case LUA_TLNGSTR: { + size = sizestring(gco2ts(o)); + break; /* nothing else to mark; make it black */ } case LUA_TUSERDATA: { Table *mt = gco2u(o)->metatable; markobject(g, mt); markobject(g, gco2u(o)->env); - gray2black(o); /* all pointers marked */ - return; + size = sizeudata(gco2u(o)); + break; } case LUA_TUPVAL: { UpVal *uv = gco2uv(o); markvalue(g, uv->v); - if (uv->v == &uv->u.value) /* closed? (open upvalues remain gray) */ - gray2black(o); /* make it black */ + if (uv->v != &uv->u.value) /* open? */ + return; /* open upvalues remain gray */ + size = sizeof(UpVal); + break; + } + case LUA_TLCL: { + gco2lcl(o)->gclist = g->gray; + g->gray = o; return; } - case LUA_TFUNCTION: { - gco2cl(o)->c.gclist = g->gray; + case LUA_TCCL: { + gco2ccl(o)->gclist = g->gray; g->gray = o; - break; + return; } case LUA_TTABLE: { linktable(gco2t(o), &g->gray); - break; + return; } case LUA_TTHREAD: { gco2th(o)->gclist = g->gray; g->gray = o; - break; + return; } case LUA_TPROTO: { gco2p(o)->gclist = g->gray; g->gray = o; - break; + return; } - default: lua_assert(0); + default: lua_assert(0); return; } + gray2black(o); + g->GCmemtrav += size; } @@ -359,7 +373,7 @@ static void traverseweakvalue (global_State *g, Table *h) { else { lua_assert(!ttisnil(gkey(n))); markvalue(g, gkey(n)); /* mark key */ - if (!hasclears && iscleared(gval(n))) /* is there a white value? */ + if (!hasclears && iscleared(g, gval(n))) /* is there a white value? */ hasclears = 1; /* table will have to be cleared */ } } @@ -388,7 +402,7 @@ static int traverseephemeron (global_State *g, Table *h) { checkdeadkey(n); if (ttisnil(gval(n))) /* entry is empty? */ removeentry(n); /* remove it */ - else if (iscleared(gkey(n))) { /* key is not marked (yet)? */ + else if (iscleared(g, gkey(n))) { /* key is not marked (yet)? */ hasclears = 1; /* table must be cleared */ if (valiswhite(gval(n))) /* value not marked yet? */ prop = 1; /* must propagate again */ @@ -426,30 +440,26 @@ static void traversestrongtable (global_State *g, Table *h) { } -static int traversetable (global_State *g, Table *h) { +static lu_mem traversetable (global_State *g, Table *h) { + const char *weakkey, *weakvalue; const TValue *mode = gfasttm(g, h->metatable, TM_MODE); markobject(g, h->metatable); - if (mode && ttisstring(mode)) { /* is there a weak mode? */ - int weakkey = (strchr(svalue(mode), 'k') != NULL); - int weakvalue = (strchr(svalue(mode), 'v') != NULL); - if (weakkey || weakvalue) { /* is really weak? */ - black2gray(obj2gco(h)); /* keep table gray */ - if (!weakkey) { /* strong keys? */ - traverseweakvalue(g, h); - return TRAVCOST + sizenode(h); - } - else if (!weakvalue) { /* strong values? */ - traverseephemeron(g, h); - return TRAVCOST + h->sizearray + sizenode(h); - } - else { - linktable(h, &g->allweak); /* nothing to traverse now */ - return TRAVCOST; - } - } /* else go through */ + if (mode && ttisstring(mode) && /* is there a weak mode? */ + ((weakkey = strchr(svalue(mode), 'k')), + (weakvalue = strchr(svalue(mode), 'v')), + (weakkey || weakvalue))) { /* is really weak? */ + black2gray(obj2gco(h)); /* keep table gray */ + if (!weakkey) /* strong keys? */ + traverseweakvalue(g, h); + else if (!weakvalue) /* strong values? */ + traverseephemeron(g, h); + else /* all weak */ + linktable(h, &g->allweak); /* nothing to traverse now */ } - traversestrongtable(g, h); - return TRAVCOST + h->sizearray + (2 * sizenode(h)); + else /* not weak */ + traversestrongtable(g, h); + return sizeof(Table) + sizeof(TValue) * h->sizearray + + sizeof(Node) * sizenode(h); } @@ -457,86 +467,101 @@ static int traverseproto (global_State *g, Proto *f) { int i; if (f->cache && iswhite(obj2gco(f->cache))) f->cache = NULL; /* allow cache to be collected */ - stringmark(f->source); + markobject(g, f->source); for (i = 0; i < f->sizek; i++) /* mark literals */ markvalue(g, &f->k[i]); for (i = 0; i < f->sizeupvalues; i++) /* mark upvalue names */ - stringmark(f->upvalues[i].name); + markobject(g, f->upvalues[i].name); for (i = 0; i < f->sizep; i++) /* mark nested protos */ markobject(g, f->p[i]); for (i = 0; i < f->sizelocvars; i++) /* mark local-variable names */ - stringmark(f->locvars[i].varname); - return TRAVCOST + f->sizek + f->sizeupvalues + f->sizep + f->sizelocvars; + markobject(g, f->locvars[i].varname); + return sizeof(Proto) + sizeof(Instruction) * f->sizecode + + sizeof(Proto *) * f->sizep + + sizeof(TValue) * f->sizek + + sizeof(int) * f->sizelineinfo + + sizeof(LocVar) * f->sizelocvars + + sizeof(Upvaldesc) * f->sizeupvalues; } -static int traverseclosure (global_State *g, Closure *cl) { - if (cl->c.isC) { - int i; - for (i=0; ic.nupvalues; i++) /* mark its upvalues */ - markvalue(g, &cl->c.upvalue[i]); - } - else { - int i; - lua_assert(cl->l.nupvalues == cl->l.p->sizeupvalues); - markobject(g, cl->l.p); /* mark its prototype */ - for (i=0; il.nupvalues; i++) /* mark its upvalues */ - markobject(g, cl->l.upvals[i]); - } - return TRAVCOST + cl->c.nupvalues; +static lu_mem traverseCclosure (global_State *g, CClosure *cl) { + int i; + for (i = 0; i < cl->nupvalues; i++) /* mark its upvalues */ + markvalue(g, &cl->upvalue[i]); + return sizeCclosure(cl->nupvalues); +} + +static lu_mem traverseLclosure (global_State *g, LClosure *cl) { + int i; + markobject(g, cl->p); /* mark its prototype */ + for (i = 0; i < cl->nupvalues; i++) /* mark its upvalues */ + markobject(g, cl->upvals[i]); + return sizeLclosure(cl->nupvalues); } -static int traversestack (global_State *g, lua_State *L) { - StkId o = L->stack; +static lu_mem traversestack (global_State *g, lua_State *th) { + StkId o = th->stack; if (o == NULL) return 1; /* stack not completely built yet */ - for (; o < L->top; o++) + for (; o < th->top; o++) markvalue(g, o); if (g->gcstate == GCSatomic) { /* final traversal? */ - StkId lim = L->stack + L->stacksize; /* real end of stack */ + StkId lim = th->stack + th->stacksize; /* real end of stack */ for (; o < lim; o++) /* clear not-marked stack slice */ setnilvalue(o); } - return TRAVCOST + cast_int(o - L->stack); + return sizeof(lua_State) + sizeof(TValue) * th->stacksize; } /* ** traverse one gray object, turning it to black (except for threads, ** which are always gray). -** Returns number of values traversed. */ -static int propagatemark (global_State *g) { +static void propagatemark (global_State *g) { + lu_mem size; GCObject *o = g->gray; lua_assert(isgray(o)); gray2black(o); switch (gch(o)->tt) { case LUA_TTABLE: { Table *h = gco2t(o); - g->gray = h->gclist; - return traversetable(g, h); + g->gray = h->gclist; /* remove from 'gray' list */ + size = traversetable(g, h); + break; + } + case LUA_TLCL: { + LClosure *cl = gco2lcl(o); + g->gray = cl->gclist; /* remove from 'gray' list */ + size = traverseLclosure(g, cl); + break; } - case LUA_TFUNCTION: { - Closure *cl = gco2cl(o); - g->gray = cl->c.gclist; - return traverseclosure(g, cl); + case LUA_TCCL: { + CClosure *cl = gco2ccl(o); + g->gray = cl->gclist; /* remove from 'gray' list */ + size = traverseCclosure(g, cl); + break; } case LUA_TTHREAD: { lua_State *th = gco2th(o); - g->gray = th->gclist; + g->gray = th->gclist; /* remove from 'gray' list */ th->gclist = g->grayagain; - g->grayagain = o; + g->grayagain = o; /* insert into 'grayagain' list */ black2gray(o); - return traversestack(g, th); + size = traversestack(g, th); + break; } case LUA_TPROTO: { Proto *p = gco2p(o); - g->gray = p->gclist; - return traverseproto(g, p); + g->gray = p->gclist; /* remove from 'gray' list */ + size = traverseproto(g, p); + break; } - default: lua_assert(0); return 0; + default: lua_assert(0); return; } + g->GCmemtrav += size; } @@ -599,12 +624,12 @@ static void convergeephemerons (global_State *g) { ** clear entries with unmarked keys from all weaktables in list 'l' up ** to element 'f' */ -static void clearkeys (GCObject *l, GCObject *f) { +static void clearkeys (global_State *g, GCObject *l, GCObject *f) { for (; l != f; l = gco2t(l)->gclist) { Table *h = gco2t(l); Node *n, *limit = gnodelast(h); for (n = gnode(h, 0); n < limit; n++) { - if (!ttisnil(gval(n)) && (iscleared(gkey(n)))) { + if (!ttisnil(gval(n)) && (iscleared(g, gkey(n)))) { setnilvalue(gval(n)); /* remove value ... */ removeentry(n); /* and remove entry from table */ } @@ -617,18 +642,18 @@ static void clearkeys (GCObject *l, GCObject *f) { ** clear entries with unmarked values from all weaktables in list 'l' up ** to element 'f' */ -static void clearvalues (GCObject *l, GCObject *f) { +static void clearvalues (global_State *g, GCObject *l, GCObject *f) { for (; l != f; l = gco2t(l)->gclist) { Table *h = gco2t(l); Node *n, *limit = gnodelast(h); int i; for (i = 0; i < h->sizearray; i++) { TValue *o = &h->array[i]; - if (iscleared(o)) /* value was collected? */ + if (iscleared(g, o)) /* value was collected? */ setnilvalue(o); /* remove value */ } for (n = gnode(h, 0); n < limit; n++) { - if (!ttisnil(gval(n)) && iscleared(gval(n))) { + if (!ttisnil(gval(n)) && iscleared(g, gval(n))) { setnilvalue(gval(n)); /* remove value ... */ removeentry(n); /* and remove entry from table */ } @@ -640,13 +665,22 @@ static void clearvalues (GCObject *l, GCObject *f) { static void freeobj (lua_State *L, GCObject *o) { switch (gch(o)->tt) { case LUA_TPROTO: luaF_freeproto(L, gco2p(o)); break; - case LUA_TFUNCTION: luaF_freeclosure(L, gco2cl(o)); break; + case LUA_TLCL: { + luaM_freemem(L, o, sizeLclosure(gco2lcl(o)->nupvalues)); + break; + } + case LUA_TCCL: { + luaM_freemem(L, o, sizeCclosure(gco2ccl(o)->nupvalues)); + break; + } case LUA_TUPVAL: luaF_freeupval(L, gco2uv(o)); break; case LUA_TTABLE: luaH_free(L, gco2t(o)); break; case LUA_TTHREAD: luaE_freethread(L, gco2th(o)); break; case LUA_TUSERDATA: luaM_freemem(L, o, sizeudata(gco2u(o))); break; - case LUA_TSTRING: { + case LUA_TSHRSTR: G(L)->strt.nuse--; + /* go through */ + case LUA_TLNGSTR: { luaM_freemem(L, o, sizestring(gco2ts(o))); break; } @@ -689,7 +723,6 @@ static GCObject **sweeplist (lua_State *L, GCObject **p, lu_mem count) { int ow = otherwhite(g); int toclear, toset; /* bits to clear and to set in all live objects */ int tostop; /* stop sweep when this is true */ - l_mem debt = g->GCdebt; /* current debt */ if (isgenerational(g)) { /* generational mode? */ toclear = ~0; /* clear nothing */ toset = bitmask(OLDBIT); /* set the old bit of all surviving objects */ @@ -708,19 +741,30 @@ static GCObject **sweeplist (lua_State *L, GCObject **p, lu_mem count) { freeobj(L, curr); /* erase 'curr' */ } else { + if (testbits(marked, tostop)) + return NULL; /* stop sweeping this list */ if (gch(curr)->tt == LUA_TTHREAD) sweepthread(L, gco2th(curr)); /* sweep thread's upvalues */ - if (testbits(marked, tostop)) { - static GCObject *nullp = NULL; - p = &nullp; /* stop sweeping this list */ - break; - } /* update marks */ gch(curr)->marked = cast_byte((marked & toclear) | toset); p = &gch(curr)->next; /* go to next element */ } } - luaE_setdebt(g, debt); /* sweeping should not change debt */ + return (*p == NULL) ? NULL : p; +} + + +/* +** sweep a list until a live object (or end of list) +*/ +static GCObject **sweeptolive (lua_State *L, GCObject **p, int *n) { + GCObject ** old = p; + int i = 0; + do { + i++; + p = sweeplist(L, p, 1); + } while (p == old); + if (n) *n += i; return p; } @@ -783,12 +827,14 @@ static void GCTM (lua_State *L, int propagateerrors) { L->allowhook = oldah; /* restore hooks */ g->gcrunning = running; /* restore state */ if (status != LUA_OK && propagateerrors) { /* error while running __gc? */ - if (status == LUA_ERRRUN) { /* is there an error msg.? */ - luaO_pushfstring(L, "error in __gc metamethod (%s)", - lua_tostring(L, -1)); + if (status == LUA_ERRRUN) { /* is there an error object? */ + const char *msg = (ttisstring(L->top - 1)) + ? svalue(L->top - 1) + : "no message"; + luaO_pushfstring(L, "error in __gc metamethod (%s)", msg); status = LUA_ERRGCMM; /* error in __gc metamethod */ } - luaD_throw(L, status); /* re-send error */ + luaD_throw(L, status); /* re-throw error */ } } } @@ -834,12 +880,21 @@ void luaC_checkfinalizer (lua_State *L, GCObject *o, Table *mt) { return; /* nothing to be done */ else { /* move 'o' to 'finobj' list */ GCObject **p; - for (p = &g->allgc; *p != o; p = &gch(*p)->next) ; - *p = gch(o)->next; /* remove 'o' from root list */ - gch(o)->next = g->finobj; /* link it in list 'finobj' */ + GCheader *ho = gch(o); + if (g->sweepgc == &ho->next) { /* avoid removing current sweep object */ + lua_assert(issweepphase(g)); + g->sweepgc = sweeptolive(L, g->sweepgc, NULL); + } + /* search for pointer pointing to 'o' */ + for (p = &g->allgc; *p != o; p = &gch(*p)->next) { /* empty */ } + *p = ho->next; /* remove 'o' from root list */ + ho->next = g->finobj; /* link it in list 'finobj' */ g->finobj = o; - l_setbit(gch(o)->marked, SEPARATED); /* mark it as such */ - resetoldbit(o); /* see MOVE OLD rule */ + l_setbit(ho->marked, SEPARATED); /* mark it as such */ + if (!keepinvariant(g)) /* not keeping invariant? */ + makewhite(g, o); /* "sweep" object */ + else + resetoldbit(o); /* see MOVE OLD rule */ } } @@ -856,6 +911,28 @@ void luaC_checkfinalizer (lua_State *L, GCObject *o, Table *mt) { #define sweepphases \ (bitmask(GCSsweepstring) | bitmask(GCSsweepudata) | bitmask(GCSsweep)) + +/* +** enter first sweep phase (strings) and prepare pointers for other +** sweep phases. The calls to 'sweeptolive' make pointers point to an +** object inside the list (instead of to the header), so that the real +** sweep do not need to skip objects created between "now" and the start +** of the real sweep. +** Returns how many objects it sweeped. +*/ +static int entersweep (lua_State *L) { + global_State *g = G(L); + int n = 0; + g->gcstate = GCSsweepstring; + lua_assert(g->sweepgc == NULL && g->sweepfin == NULL); + /* prepare to sweep strings, finalizable objects, and regular objects */ + g->sweepstrgc = 0; + g->sweepfin = sweeptolive(L, &g->finobj, &n); + g->sweepgc = sweeptolive(L, &g->allgc, &n); + return n; +} + + /* ** change GC mode */ @@ -865,15 +942,14 @@ void luaC_changemode (lua_State *L, int mode) { if (mode == KGC_GEN) { /* change to generational mode */ /* make sure gray lists are consistent */ luaC_runtilstate(L, bitmask(GCSpropagate)); - g->lastmajormem = gettotalbytes(g); + g->GCestimate = gettotalbytes(g); g->gckind = KGC_GEN; } else { /* change to incremental mode */ /* sweep all objects to turn them back to white (as white has not changed, nothing extra will be collected) */ - g->sweepstrgc = 0; - g->gcstate = GCSsweepstring; g->gckind = KGC_NORMAL; + entersweep(L); luaC_runtilstate(L, ~sweepphases); } } @@ -907,8 +983,9 @@ void luaC_freeallobjects (lua_State *L) { } -static void atomic (lua_State *L) { +static l_mem atomic (lua_State *L) { global_State *g = G(L); + l_mem work = -g->GCmemtrav; /* start counting work */ GCObject *origweak, *origall; lua_assert(!iswhite(obj2gco(g->mainthread))); markobject(g, L); /* mark running thread */ @@ -917,77 +994,87 @@ static void atomic (lua_State *L) { markmt(g); /* mark basic metatables */ /* remark occasional upvalues of (maybe) dead threads */ remarkupvals(g); + propagateall(g); /* propagate changes */ + work += g->GCmemtrav; /* stop counting (do not (re)count grays) */ /* traverse objects caught by write barrier and by 'remarkupvals' */ retraversegrays(g); + work -= g->GCmemtrav; /* restart counting */ convergeephemerons(g); /* at this point, all strongly accessible objects are marked. */ /* clear values from weak tables, before checking finalizers */ - clearvalues(g->weak, NULL); - clearvalues(g->allweak, NULL); + clearvalues(g, g->weak, NULL); + clearvalues(g, g->allweak, NULL); origweak = g->weak; origall = g->allweak; + work += g->GCmemtrav; /* stop counting (objects being finalized) */ separatetobefnz(L, 0); /* separate objects to be finalized */ - markbeingfnz(g); /* mark userdata that will be finalized */ + markbeingfnz(g); /* mark objects that will be finalized */ propagateall(g); /* remark, to propagate `preserveness' */ + work -= g->GCmemtrav; /* restart counting */ convergeephemerons(g); /* at this point, all resurrected objects are marked. */ /* remove dead objects from weak tables */ - clearkeys(g->ephemeron, NULL); /* clear keys from all ephemeron tables */ - clearkeys(g->allweak, NULL); /* clear keys from all allweak tables */ + clearkeys(g, g->ephemeron, NULL); /* clear keys from all ephemeron tables */ + clearkeys(g, g->allweak, NULL); /* clear keys from all allweak tables */ /* clear values from resurrected weak tables */ - clearvalues(g->weak, origweak); - clearvalues(g->allweak, origall); - g->sweepstrgc = 0; /* prepare to sweep strings */ - g->gcstate = GCSsweepstring; + clearvalues(g, g->weak, origweak); + clearvalues(g, g->allweak, origall); g->currentwhite = cast_byte(otherwhite(g)); /* flip current white */ - /*lua_checkmemory(L);*/ + work += g->GCmemtrav; /* complete counting */ + return work; /* estimate of memory marked by 'atomic' */ } -static l_mem singlestep (lua_State *L) { +static lu_mem singlestep (lua_State *L) { global_State *g = G(L); switch (g->gcstate) { case GCSpause: { + g->GCmemtrav = 0; /* start to count memory traversed */ if (!isgenerational(g)) markroot(g); /* start a new collection */ - /* in any case, root must be marked */ + /* in any case, root must be marked at this point */ lua_assert(!iswhite(obj2gco(g->mainthread)) && !iswhite(gcvalue(&g->l_registry))); g->gcstate = GCSpropagate; - return GCROOTCOST; + return g->GCmemtrav; } case GCSpropagate: { - if (g->gray) - return propagatemark(g); + if (g->gray) { + lu_mem oldtrav = g->GCmemtrav; + propagatemark(g); + return g->GCmemtrav - oldtrav; /* memory traversed in this step */ + } else { /* no more `gray' objects */ + lu_mem work; + int sw; g->gcstate = GCSatomic; /* finish mark phase */ - atomic(L); - return GCATOMICCOST; + g->GCestimate = g->GCmemtrav; /* save what was counted */; + work = atomic(L); /* add what was traversed by 'atomic' */ + g->GCestimate += work; /* estimate of total memory traversed */ + sw = entersweep(L); + return work + sw * GCSWEEPCOST; } } case GCSsweepstring: { - if (g->sweepstrgc < g->strt.size) { - sweepwholelist(L, &g->strt.hash[g->sweepstrgc++]); - return GCSWEEPCOST; - } - else { /* no more strings to sweep */ - g->sweepgc = &g->finobj; /* prepare to sweep finalizable objects */ + int i; + for (i = 0; i < GCSWEEPMAX && g->sweepstrgc + i < g->strt.size; i++) + sweepwholelist(L, &g->strt.hash[g->sweepstrgc + i]); + g->sweepstrgc += i; + if (g->sweepstrgc >= g->strt.size) /* no more strings to sweep? */ g->gcstate = GCSsweepudata; - return 0; - } + return i * GCSWEEPCOST; } case GCSsweepudata: { - if (*g->sweepgc) { - g->sweepgc = sweeplist(L, g->sweepgc, GCSWEEPMAX); + if (g->sweepfin) { + g->sweepfin = sweeplist(L, g->sweepfin, GCSWEEPMAX); return GCSWEEPMAX*GCSWEEPCOST; } else { - g->sweepgc = &g->allgc; /* go to next phase */ g->gcstate = GCSsweep; - return GCSWEEPCOST; + return 0; } } case GCSsweep: { - if (*g->sweepgc) { + if (g->sweepgc) { g->sweepgc = sweeplist(L, g->sweepgc, GCSWEEPMAX); return GCSWEEPMAX*GCSWEEPCOST; } @@ -1018,43 +1105,52 @@ void luaC_runtilstate (lua_State *L, int statesmask) { static void generationalcollection (lua_State *L) { global_State *g = G(L); - if (g->lastmajormem == 0) { /* signal for another major collection? */ + if (g->GCestimate == 0) { /* signal for another major collection? */ luaC_fullgc(L, 0); /* perform a full regular collection */ - g->lastmajormem = gettotalbytes(g); /* update control */ + g->GCestimate = gettotalbytes(g); /* update control */ } else { + lu_mem estimate = g->GCestimate; luaC_runtilstate(L, ~bitmask(GCSpause)); /* run complete cycle */ luaC_runtilstate(L, bitmask(GCSpause)); - if (gettotalbytes(g) > g->lastmajormem/100 * g->gcmajorinc) - g->lastmajormem = 0; /* signal for a major collection */ + if (gettotalbytes(g) > (estimate / 100) * g->gcmajorinc) + g->GCestimate = 0; /* signal for a major collection */ } luaE_setdebt(g, stddebt(g)); } -static void step (lua_State *L) { +static void incstep (lua_State *L) { global_State *g = G(L); - l_mem lim = g->gcstepmul; /* how much to work */ + l_mem debt = g->GCdebt; + int stepmul = g->gcstepmul; + if (stepmul < 40) stepmul = 40; /* avoid ridiculous low values */ + /* convert debt from Kb to 'work units' (avoid zero debt and overflows) */ + debt = (debt / STEPMULADJ) + 1; + debt = (debt < MAX_LMEM / stepmul) ? debt * stepmul : MAX_LMEM; do { /* always perform at least one single step */ - lim -= singlestep(L); - } while (lim > 0 && g->gcstate != GCSpause); - if (g->gcstate != GCSpause) - luaE_setdebt(g, g->GCdebt - GCSTEPSIZE); + lu_mem work = singlestep(L); /* do some work */ + debt -= work; + } while (debt > -GCSTEPSIZE && g->gcstate != GCSpause); + if (g->gcstate == GCSpause) + debt = stddebtest(g, g->GCestimate); /* pause until next cycle */ else - luaE_setdebt(g, stddebt(g)); + debt = (debt / stepmul) * STEPMULADJ; /* convert 'work units' to Kb */ + luaE_setdebt(g, debt); } /* -** performs a basic GC step even if the collector is stopped +** performs a basic GC step */ void luaC_forcestep (lua_State *L) { global_State *g = G(L); int i; if (isgenerational(g)) generationalcollection(L); - else step(L); - for (i = 0; i < GCFINALIZENUM && g->tobefnz; i++) - GCTM(L, 1); /* Call a few pending finalizers */ + else incstep(L); + /* run a few finalizers (or all of them at the end of a collect cycle) */ + for (i = 0; g->tobefnz && (i < GCFINALIZENUM || g->gcstate == GCSpause); i++) + GCTM(L, 1); /* call one finalizer */ } @@ -1062,10 +1158,13 @@ void luaC_forcestep (lua_State *L) { ** performs a basic GC step only if collector is running */ void luaC_step (lua_State *L) { - if (G(L)->gcrunning) luaC_forcestep(L); + global_State *g = G(L); + if (g->gcrunning) luaC_forcestep(L); + else luaE_setdebt(g, -GCSTEPSIZE); /* avoid being called too often */ } + /* ** performs a full GC cycle; if "isemergency", does not call ** finalizers (which could change stack positions) @@ -1073,16 +1172,19 @@ void luaC_step (lua_State *L) { void luaC_fullgc (lua_State *L, int isemergency) { global_State *g = G(L); int origkind = g->gckind; + int someblack = keepinvariant(g); lua_assert(origkind != KGC_EMERGENCY); - if (!isemergency) /* do not run finalizers during emergency GC */ + if (isemergency) /* do not run finalizers during emergency GC */ + g->gckind = KGC_EMERGENCY; + else { + g->gckind = KGC_NORMAL; callallpendingfinalizers(L, 1); - if (keepinvariant(g)) { /* marking phase? */ + } + if (someblack) { /* may there be some black objects? */ /* must sweep all objects to turn them back to white (as white has not changed, nothing will be collected) */ - g->sweepstrgc = 0; - g->gcstate = GCSsweepstring; + entersweep(L); } - g->gckind = isemergency ? KGC_EMERGENCY : KGC_NORMAL; /* finish any pending sweep phase to start a new cycle */ luaC_runtilstate(L, bitmask(GCSpause)); /* run entire collector */ diff --git a/depends/lua/src/llex.c b/depends/lua/src/llex.c index 74deebb93..c4d8c65b6 100644 --- a/depends/lua/src/llex.c +++ b/depends/lua/src/llex.c @@ -1,5 +1,5 @@ /* -** $Id: llex.c,v 2.59 2011/11/30 12:43:51 roberto Exp $ +** $Id: llex.c,v 2.61 2012/01/23 23:05:51 roberto Exp $ ** Lexical Analyzer ** See Copyright Notice in lua.h */ @@ -67,7 +67,7 @@ void luaX_init (lua_State *L) { for (i=0; itsv.reserved = cast_byte(i+1); /* reserved word */ + ts->tsv.extra = cast_byte(i+1); /* reserved word */ } } @@ -222,13 +222,24 @@ static void trydecpoint (LexState *ls, SemInfo *seminfo) { /* LUA_NUMBER */ +/* +** this function is quite liberal in what it accepts, as 'luaO_str2d' +** will reject ill-formed numerals. +*/ static void read_numeral (LexState *ls, SemInfo *seminfo) { + const char *expo = "Ee"; + int first = ls->current; lua_assert(lisdigit(ls->current)); - do { - save_and_next(ls); - if (check_next(ls, "EePp")) /* exponent part? */ + save_and_next(ls); + if (first == '0' && check_next(ls, "Xx")) /* hexadecimal? */ + expo = "Pp"; + for (;;) { + if (check_next(ls, expo)) /* exponent part? */ check_next(ls, "+-"); /* optional exponent sign */ - } while (lislalnum(ls->current) || ls->current == '.'); + if (lisxdigit(ls->current) || ls->current == '.') + save_and_next(ls); + else break; + } save(ls, '\0'); buffreplace(ls, '.', ls->decpoint); /* follow locale for decimal point */ if (!buff2d(ls->buff, &seminfo->r)) /* format error? */ @@ -480,8 +491,8 @@ static int llex (LexState *ls, SemInfo *seminfo) { ts = luaX_newstring(ls, luaZ_buffer(ls->buff), luaZ_bufflen(ls->buff)); seminfo->ts = ts; - if (ts->tsv.reserved > 0) /* reserved word? */ - return ts->tsv.reserved - 1 + FIRST_RESERVED; + if (isreserved(ts)) /* reserved word? */ + return ts->tsv.extra - 1 + FIRST_RESERVED; else { return TK_NAME; } diff --git a/depends/lua/src/lmathlib.c b/depends/lua/src/lmathlib.c index b17237f0b..c3c605e86 100644 --- a/depends/lua/src/lmathlib.c +++ b/depends/lua/src/lmathlib.c @@ -1,5 +1,5 @@ /* -** $Id: lmathlib.c,v 1.80 2011/07/05 12:49:35 roberto Exp $ +** $Id: lmathlib.c,v 1.81 2012/05/18 17:47:53 roberto Exp $ ** Standard mathematical library ** See Copyright Notice in lua.h */ @@ -17,17 +17,17 @@ #include "lualib.h" -#undef PI -#define PI (3.14159265358979323846) -#define RADIANS_PER_DEGREE (PI/180.0) - - /* macro 'l_tg' allows the addition of an 'l' or 'f' to all math operations */ #if !defined(l_tg) #define l_tg(x) (x) #endif +#undef PI +#define PI (l_tg(3.1415926535897932384626433832795)) +#define RADIANS_PER_DEGREE (PI/180.0) + + static int math_abs (lua_State *L) { lua_pushnumber(L, l_tg(fabs)(luaL_checknumber(L, 1))); diff --git a/depends/lua/src/lmem.c b/depends/lua/src/lmem.c index 792deb3cf..3f88496e0 100644 --- a/depends/lua/src/lmem.c +++ b/depends/lua/src/lmem.c @@ -1,5 +1,5 @@ /* -** $Id: lmem.c,v 1.83 2011/11/30 12:42:49 roberto Exp $ +** $Id: lmem.c,v 1.84 2012/05/23 15:41:53 roberto Exp $ ** Interface to Memory Manager ** See Copyright Notice in lua.h */ @@ -94,22 +94,6 @@ void *luaM_realloc_ (lua_State *L, void *block, size_t osize, size_t nsize) { } lua_assert((nsize == 0) == (newblock == NULL)); g->GCdebt = (g->GCdebt + nsize) - realosize; -#if defined(TRACEMEM) - { /* auxiliary patch to monitor garbage collection. - ** To plot, gnuplot with following command: - ** plot TRACEMEM using 1:2 with lines, TRACEMEM using 1:3 with lines - */ - static unsigned long total = 0; /* our "time" */ - static FILE *f = NULL; /* output file */ - total++; /* "time" always grows */ - if ((total % 200) == 0) { - if (f == NULL) f = fopen(TRACEMEM, "w"); - fprintf(f, "%lu %u %d %d\n", total, - gettotalbytes(g), g->GCdebt, g->gcstate * 10000); - } - } -#endif - return newblock; } diff --git a/depends/lua/src/loadlib.c b/depends/lua/src/loadlib.c index 783bc12be..a9959277b 100644 --- a/depends/lua/src/loadlib.c +++ b/depends/lua/src/loadlib.c @@ -1,5 +1,5 @@ /* -** $Id: loadlib.c,v 1.108 2011/12/12 16:34:03 roberto Exp $ +** $Id: loadlib.c,v 1.111 2012/05/30 12:33:44 roberto Exp $ ** Dynamic library loader for Lua ** See Copyright Notice in lua.h ** @@ -92,9 +92,9 @@ #define LUA_OFSEP "_" -#define LIBPREFIX "LOADLIB: " +/* table (in the registry) that keeps handles for all loaded C libraries */ +#define CLIBS "_CLIBS" -#define POF LUA_POF #define LIB_FAIL "open" @@ -248,48 +248,54 @@ static lua_CFunction ll_sym (lua_State *L, void *lib, const char *sym) { #endif - -static void **ll_register (lua_State *L, const char *path) { - void **plib; - lua_pushfstring(L, "%s%s", LIBPREFIX, path); - lua_gettable(L, LUA_REGISTRYINDEX); /* check library in registry? */ - if (!lua_isnil(L, -1)) /* is there an entry? */ - plib = (void **)lua_touserdata(L, -1); - else { /* no entry yet; create one */ - lua_pop(L, 1); /* remove result from gettable */ - plib = (void **)lua_newuserdata(L, sizeof(const void *)); - *plib = NULL; - luaL_setmetatable(L, "_LOADLIB"); - lua_pushfstring(L, "%s%s", LIBPREFIX, path); - lua_pushvalue(L, -2); - lua_settable(L, LUA_REGISTRYINDEX); - } +static void *ll_checkclib (lua_State *L, const char *path) { + void *plib; + lua_getfield(L, LUA_REGISTRYINDEX, CLIBS); + lua_getfield(L, -1, path); + plib = lua_touserdata(L, -1); /* plib = CLIBS[path] */ + lua_pop(L, 2); /* pop CLIBS table and 'plib' */ return plib; } +static void ll_addtoclib (lua_State *L, const char *path, void *plib) { + lua_getfield(L, LUA_REGISTRYINDEX, CLIBS); + lua_pushlightuserdata(L, plib); + lua_pushvalue(L, -1); + lua_setfield(L, -3, path); /* CLIBS[path] = plib */ + lua_rawseti(L, -2, luaL_len(L, -2) + 1); /* CLIBS[#CLIBS + 1] = plib */ + lua_pop(L, 1); /* pop CLIBS table */ +} + + /* -** __gc tag method: calls library's `ll_unloadlib' function with the lib -** handle +** __gc tag method for CLIBS table: calls 'll_unloadlib' for all lib +** handles in list CLIBS */ static int gctm (lua_State *L) { - void **lib = (void **)luaL_checkudata(L, 1, "_LOADLIB"); - if (*lib) ll_unloadlib(*lib); - *lib = NULL; /* mark library as closed */ + int n = luaL_len(L, 1); + for (; n >= 1; n--) { /* for each handle, in reverse order */ + lua_rawgeti(L, 1, n); /* get handle CLIBS[n] */ + ll_unloadlib(lua_touserdata(L, -1)); + lua_pop(L, 1); /* pop handle */ + } return 0; } static int ll_loadfunc (lua_State *L, const char *path, const char *sym) { - void **reg = ll_register(L, path); - if (*reg == NULL) *reg = ll_load(L, path, *sym == '*'); - if (*reg == NULL) return ERRLIB; /* unable to load library */ + void *reg = ll_checkclib(L, path); /* check loaded C libraries */ + if (reg == NULL) { /* must load library? */ + reg = ll_load(L, path, *sym == '*'); + if (reg == NULL) return ERRLIB; /* unable to load library */ + ll_addtoclib(L, path, reg); + } if (*sym == '*') { /* loading only library (no function)? */ lua_pushboolean(L, 1); /* return 'true' */ return 0; /* no errors */ } else { - lua_CFunction f = ll_sym(L, *reg, sym); + lua_CFunction f = ll_sym(L, reg, sym); if (f == NULL) return ERRFUNC; /* unable to find function */ lua_pushcfunction(L, f); /* else create new function */ @@ -418,12 +424,12 @@ static int loadfunc (lua_State *L, const char *filename, const char *modname) { if (mark) { int stat; funcname = lua_pushlstring(L, modname, mark - modname); - funcname = lua_pushfstring(L, POF"%s", funcname); + funcname = lua_pushfstring(L, LUA_POF"%s", funcname); stat = ll_loadfunc(L, filename, funcname); if (stat != ERRFUNC) return stat; modname = mark + 1; /* else go ahead and try old-style name */ } - funcname = lua_pushfstring(L, POF"%s", modname); + funcname = lua_pushfstring(L, LUA_POF"%s", modname); return ll_loadfunc(L, filename, funcname); } @@ -476,9 +482,9 @@ static void findloader (lua_State *L, const char *name) { lua_getfield(L, lua_upvalueindex(1), "searchers"); /* will be at index 3 */ if (!lua_istable(L, 3)) luaL_error(L, LUA_QL("package.searchers") " must be a table"); - /* iterate over available seachers to find a loader */ + /* iterate over available searchers to find a loader */ for (i = 1; ; i++) { - lua_rawgeti(L, 3, i); /* get a seacher */ + lua_rawgeti(L, 3, i); /* get a searcher */ if (lua_isnil(L, -1)) { /* no more searchers? */ lua_pop(L, 1); /* remove nil */ luaL_pushresult(&msg); /* create error message */ @@ -666,18 +672,10 @@ static const luaL_Reg ll_funcs[] = { }; -static const lua_CFunction searchers[] = - {searcher_preload, searcher_Lua, searcher_C, searcher_Croot, NULL}; - - -LUAMOD_API int luaopen_package (lua_State *L) { +static void createsearcherstable (lua_State *L) { + static const lua_CFunction searchers[] = + {searcher_preload, searcher_Lua, searcher_C, searcher_Croot, NULL}; int i; - /* create new type _LOADLIB */ - luaL_newmetatable(L, "_LOADLIB"); - lua_pushcfunction(L, gctm); - lua_setfield(L, -2, "__gc"); - /* create `package' table */ - luaL_newlib(L, pk_funcs); /* create 'searchers' table */ lua_createtable(L, sizeof(searchers)/sizeof(searchers[0]) - 1, 0); /* fill it with pre-defined searchers */ @@ -686,6 +684,19 @@ LUAMOD_API int luaopen_package (lua_State *L) { lua_pushcclosure(L, searchers[i], 1); lua_rawseti(L, -2, i+1); } +} + + +LUAMOD_API int luaopen_package (lua_State *L) { + /* create table CLIBS to keep track of loaded C libraries */ + luaL_getsubtable(L, LUA_REGISTRYINDEX, CLIBS); + lua_createtable(L, 0, 1); /* metatable for CLIBS */ + lua_pushcfunction(L, gctm); + lua_setfield(L, -2, "__gc"); /* set finalizer for CLIBS table */ + lua_setmetatable(L, -2); + /* create `package' table */ + luaL_newlib(L, pk_funcs); + createsearcherstable(L); #if defined(LUA_COMPAT_LOADERS) lua_pushvalue(L, -1); /* make a copy of 'searchers' table */ lua_setfield(L, -3, "loaders"); /* put it in field `loaders' */ diff --git a/depends/lua/src/lopcodes.c b/depends/lua/src/lopcodes.c index 2e3467668..ef7369275 100644 --- a/depends/lua/src/lopcodes.c +++ b/depends/lua/src/lopcodes.c @@ -1,5 +1,6 @@ /* -** $Id: lopcodes.c,v 1.48 2011/04/19 16:22:13 roberto Exp $ +** $Id: lopcodes.c,v 1.49 2012/05/14 13:34:18 roberto Exp $ +** Opcodes for Lua virtual machine ** See Copyright Notice in lua.h */ diff --git a/depends/lua/src/loslib.c b/depends/lua/src/loslib.c index 881667dac..489755b6f 100644 --- a/depends/lua/src/loslib.c +++ b/depends/lua/src/loslib.c @@ -1,5 +1,5 @@ /* -** $Id: loslib.c,v 1.38 2011/11/30 12:35:05 roberto Exp $ +** $Id: loslib.c,v 1.39 2012/05/23 15:37:09 roberto Exp $ ** Standard Operating System library ** See Copyright Notice in lua.h */ @@ -26,11 +26,12 @@ #if !defined(LUA_STRFTIMEOPTIONS) #if !defined(LUA_USE_POSIX) -#define LUA_STRFTIMEOPTIONS { "aAbBcdHIjmMpSUwWxXyYz%", "" } +#define LUA_STRFTIMEOPTIONS { "aAbBcdHIjmMpSUwWxXyYz%", "" } #else -#define LUA_STRFTIMEOPTIONS { "aAbBcCdDeFgGhHIjmMnprRStTuUVwWxXyYzZ%", "", \ - "E", "cCxXyY", \ - "O", "deHImMSuUVwWy" } +#define LUA_STRFTIMEOPTIONS \ + { "aAbBcCdDeFgGhHIjmMnprRStTuUVwWxXyYzZ%", "" \ + "", "E", "cCxXyY", \ + "O", "deHImMSuUVwWy" } #endif #endif @@ -43,7 +44,7 @@ */ #if defined(LUA_USE_MKSTEMP) #include -#define LUA_TMPNAMBUFSIZE 32 +#define LUA_TMPNAMBUFSIZE 32 #define lua_tmpnam(b,e) { \ strcpy(b, "/tmp/lua_XXXXXX"); \ e = mkstemp(b); \ @@ -52,8 +53,8 @@ #elif !defined(lua_tmpnam) -#define LUA_TMPNAMBUFSIZE L_tmpnam -#define lua_tmpnam(b,e) { e = (tmpnam(b) == NULL); } +#define LUA_TMPNAMBUFSIZE L_tmpnam +#define lua_tmpnam(b,e) { e = (tmpnam(b) == NULL); } #endif diff --git a/depends/lua/src/lparser.c b/depends/lua/src/lparser.c index 4d689365a..b3eb3ca92 100644 --- a/depends/lua/src/lparser.c +++ b/depends/lua/src/lparser.c @@ -1,5 +1,5 @@ /* -** $Id: lparser.c,v 2.124 2011/12/02 13:23:56 roberto Exp $ +** $Id: lparser.c,v 2.128 2012/05/20 14:51:23 roberto Exp $ ** Lua Parser ** See Copyright Notice in lua.h */ @@ -222,7 +222,7 @@ static int searchupvalue (FuncState *fs, TString *name) { int i; Upvaldesc *up = fs->f->upvalues; for (i = 0; i < fs->nups; i++) { - if (eqstr(up[i].name, name)) return i; + if (luaS_eqstr(up[i].name, name)) return i; } return -1; /* not found */ } @@ -246,7 +246,7 @@ static int newupvalue (FuncState *fs, TString *name, expdesc *v) { static int searchvar (FuncState *fs, TString *n) { int i; for (i=fs->nactvar-1; i >= 0; i--) { - if (eqstr(n, getlocvar(fs, i)->varname)) + if (luaS_eqstr(n, getlocvar(fs, i)->varname)) return i; } return -1; /* not found */ @@ -342,7 +342,7 @@ static void closegoto (LexState *ls, int g, Labeldesc *label) { FuncState *fs = ls->fs; Labellist *gl = &ls->dyd->gt; Labeldesc *gt = &gl->arr[g]; - lua_assert(eqstr(gt->name, label->name)); + lua_assert(luaS_eqstr(gt->name, label->name)); if (gt->nactvar < label->nactvar) { TString *vname = getlocvar(fs, gt->nactvar)->varname; const char *msg = luaO_pushfstring(ls->L, @@ -369,7 +369,7 @@ static int findlabel (LexState *ls, int g) { /* check labels in current block for a match */ for (i = bl->firstlabel; i < dyd->label.n; i++) { Labeldesc *lb = &dyd->label.arr[i]; - if (eqstr(lb->name, gt->name)) { /* correct label? */ + if (luaS_eqstr(lb->name, gt->name)) { /* correct label? */ if (gt->nactvar > lb->nactvar && (bl->upval || dyd->label.n > bl->firstlabel)) luaK_patchclose(ls->fs, gt->pc, lb->nactvar); @@ -403,7 +403,7 @@ static void findgotos (LexState *ls, Labeldesc *lb) { Labellist *gl = &ls->dyd->gt; int i = ls->fs->bl->firstgoto; while (i < gl->n) { - if (eqstr(gl->arr[i].name, lb->name)) + if (luaS_eqstr(gl->arr[i].name, lb->name)) closegoto(ls, i, lb); else i++; @@ -461,7 +461,7 @@ static void breaklabel (LexState *ls) { ** message when label name is a reserved word (which can only be 'break') */ static l_noret undefgoto (LexState *ls, Labeldesc *gt) { - const char *msg = (gt->name->tsv.reserved > 0) + const char *msg = isreserved(gt->name) ? "<%s> at line %d not inside a loop" : "no visible label " LUA_QS " for at line %d"; msg = luaO_pushfstring(ls->L, msg, getstr(gt->name), gt->line); @@ -493,21 +493,30 @@ static void leaveblock (FuncState *fs) { /* -** adds prototype being created into its parent list of prototypes -** and codes instruction to create new closure +** adds a new prototype into list of prototypes */ -static void codeclosure (LexState *ls, Proto *clp, expdesc *v) { - FuncState *fs = ls->fs->prev; - Proto *f = fs->f; /* prototype of function creating new closure */ +static Proto *addprototype (LexState *ls) { + Proto *clp; + lua_State *L = ls->L; + FuncState *fs = ls->fs; + Proto *f = fs->f; /* prototype of current function */ if (fs->np >= f->sizep) { int oldsize = f->sizep; - luaM_growvector(ls->L, f->p, fs->np, f->sizep, Proto *, - MAXARG_Bx, "functions"); + luaM_growvector(L, f->p, fs->np, f->sizep, Proto *, MAXARG_Bx, "functions"); while (oldsize < f->sizep) f->p[oldsize++] = NULL; } - f->p[fs->np++] = clp; - luaC_objbarrier(ls->L, f, clp); - init_exp(v, VRELOCABLE, luaK_codeABx(fs, OP_CLOSURE, 0, fs->np-1)); + f->p[fs->np++] = clp = luaF_newproto(L); + luaC_objbarrier(L, f, clp); + return clp; +} + + +/* +** codes instruction to create new closure in parent function +*/ +static void codeclosure (LexState *ls, expdesc *v) { + FuncState *fs = ls->fs->prev; + init_exp(v, VRELOCABLE, luaK_codeABx(fs, OP_CLOSURE, 0, fs->np - 1)); luaK_exp2nextreg(fs, v); /* fix it at stack top (for GC) */ } @@ -529,13 +538,9 @@ static void open_func (LexState *ls, FuncState *fs, BlockCnt *bl) { fs->nactvar = 0; fs->firstlocal = ls->dyd->actvar.n; fs->bl = NULL; - f = luaF_newproto(L); - fs->f = f; + f = fs->f; f->source = ls->source; f->maxstacksize = 2; /* registers 0/1 are always valid */ - /* anchor prototype (to avoid being collected) */ - setptvalue2s(L, L->top, f); - incr_top(L); fs->h = luaH_new(L); /* anchor table of constants (to avoid being collected) */ sethvalue2s(L, L->top, fs->h); @@ -568,20 +573,6 @@ static void close_func (LexState *ls) { anchor_token(ls); L->top--; /* pop table of constants */ luaC_checkGC(L); - L->top--; /* pop prototype (after possible collection) */ -} - - -/* -** opens the main function, which is a regular vararg function with an -** upvalue named LUA_ENV -*/ -static void open_mainfunc (LexState *ls, FuncState *fs, BlockCnt *bl) { - expdesc v; - open_func(ls, fs, bl); - fs->f->is_vararg = 1; /* main function is always vararg */ - init_exp(&v, VLOCAL, 0); - newupvalue(fs, ls->envn, &v); /* create environment upvalue */ } @@ -795,8 +786,9 @@ static void body (LexState *ls, expdesc *e, int ismethod, int line) { /* body -> `(' parlist `)' block END */ FuncState new_fs; BlockCnt bl; - open_func(ls, &new_fs, &bl); + new_fs.f = addprototype(ls); new_fs.f->linedefined = line; + open_func(ls, &new_fs, &bl); checknext(ls, '('); if (ismethod) { new_localvarliteral(ls, "self"); /* create 'self' parameter */ @@ -807,7 +799,7 @@ static void body (LexState *ls, expdesc *e, int ismethod, int line) { statlist(ls); new_fs.f->lastlinedefined = ls->linenumber; check_match(ls, TK_END, TK_FUNCTION, line); - codeclosure(ls, new_fs.f, e); + codeclosure(ls, e); close_func(ls); } @@ -879,8 +871,8 @@ static void funcargs (LexState *ls, expdesc *f, int line) { */ -static void prefixexp (LexState *ls, expdesc *v) { - /* prefixexp -> NAME | '(' expr ')' */ +static void primaryexp (LexState *ls, expdesc *v) { + /* primaryexp -> NAME | '(' expr ')' */ switch (ls->t.token) { case '(': { int line = ls->linenumber; @@ -901,12 +893,12 @@ static void prefixexp (LexState *ls, expdesc *v) { } -static void primaryexp (LexState *ls, expdesc *v) { - /* primaryexp -> - prefixexp { `.' NAME | `[' exp `]' | `:' NAME funcargs | funcargs } */ +static void suffixedexp (LexState *ls, expdesc *v) { + /* suffixedexp -> + primaryexp { '.' NAME | '[' exp ']' | ':' NAME funcargs | funcargs } */ FuncState *fs = ls->fs; int line = ls->linenumber; - prefixexp(ls, v); + primaryexp(ls, v); for (;;) { switch (ls->t.token) { case '.': { /* fieldsel */ @@ -941,7 +933,7 @@ static void primaryexp (LexState *ls, expdesc *v) { static void simpleexp (LexState *ls, expdesc *v) { /* simpleexp -> NUMBER | STRING | NIL | TRUE | FALSE | ... | - constructor | FUNCTION body | primaryexp */ + constructor | FUNCTION body | suffixedexp */ switch (ls->t.token) { case TK_NUMBER: { init_exp(v, VKNUM, 0); @@ -981,7 +973,7 @@ static void simpleexp (LexState *ls, expdesc *v) { return; } default: { - primaryexp(ls, v); + suffixedexp(ls, v); return; } } @@ -1141,10 +1133,10 @@ static void check_conflict (LexState *ls, struct LHS_assign *lh, expdesc *v) { static void assignment (LexState *ls, struct LHS_assign *lh, int nvars) { expdesc e; check_condition(ls, vkisvar(lh->v.k), "syntax error"); - if (testnext(ls, ',')) { /* assignment -> `,' primaryexp assignment */ + if (testnext(ls, ',')) { /* assignment -> ',' suffixedexp assignment */ struct LHS_assign nv; nv.prev = lh; - primaryexp(ls, &nv.v); + suffixedexp(ls, &nv.v); if (nv.v.k != VINDEXED) check_conflict(ls, lh, &nv.v); checklimit(ls->fs, nvars + ls->L->nCcalls, LUAI_MAXCCALLS, @@ -1200,7 +1192,7 @@ static void gotostat (LexState *ls, int pc) { static void checkrepeated (FuncState *fs, Labellist *ll, TString *label) { int i; for (i = fs->bl->firstlabel; i < ll->n; i++) { - if (eqstr(label, ll->arr[i].name)) { + if (luaS_eqstr(label, ll->arr[i].name)) { const char *msg = luaO_pushfstring(fs->ls->L, "label " LUA_QS " already defined on line %d", getstr(label), ll->arr[i].line); @@ -1210,6 +1202,13 @@ static void checkrepeated (FuncState *fs, Labellist *ll, TString *label) { } +/* skip no-op statements */ +static void skipnoopstat (LexState *ls) { + while (ls->t.token == ';' || ls->t.token == TK_DBCOLON) + statement(ls); +} + + static void labelstat (LexState *ls, TString *label, int line) { /* label -> '::' NAME '::' */ FuncState *fs = ls->fs; @@ -1219,9 +1218,7 @@ static void labelstat (LexState *ls, TString *label, int line) { checknext(ls, TK_DBCOLON); /* skip double colon */ /* create new entry for this label */ l = newlabelentry(ls, ll, label, line, fs->pc); - /* skip other no-op statements */ - while (ls->t.token == ';' || ls->t.token == TK_DBCOLON) - statement(ls); + skipnoopstat(ls); /* skip other no-op statements */ if (block_follow(ls, 0)) { /* label is last no-op statement in the block? */ /* assume that locals are already out of scope */ ll->arr[l].nactvar = fs->bl->nactvar; @@ -1384,6 +1381,7 @@ static void test_then_block (LexState *ls, int *escapelist) { luaK_goiffalse(ls->fs, &v); /* will jump to label if condition is true */ enterblock(fs, &bl, 0); /* must enter block before 'goto' */ gotostat(ls, v.t); /* handle goto/break */ + skipnoopstat(ls); /* skip other no-op statements */ if (block_follow(ls, 0)) { /* 'goto' is the entire block? */ leaveblock(fs); return; /* and that is it */ @@ -1480,13 +1478,15 @@ static void exprstat (LexState *ls) { /* stat -> func | assignment */ FuncState *fs = ls->fs; struct LHS_assign v; - primaryexp(ls, &v.v); - if (v.v.k == VCALL) /* stat -> func */ - SETARG_C(getcode(fs, &v.v), 1); /* call statement uses no results */ - else { /* stat -> assignment */ + suffixedexp(ls, &v.v); + if (ls->t.token == '=' || ls->t.token == ',') { /* stat -> assignment ? */ v.prev = NULL; assignment(ls, &v, 1); } + else { /* stat -> func */ + check_condition(ls, v.v.k == VCALL, "syntax error"); + SETARG_C(getcode(fs, &v.v), 1); /* call statement uses no results */ + } } @@ -1594,27 +1594,42 @@ static void statement (LexState *ls) { /* }====================================================================== */ -Proto *luaY_parser (lua_State *L, ZIO *z, Mbuffer *buff, - Dyndata *dyd, const char *name, int firstchar) { +/* +** compiles the main function, which is a regular vararg function with an +** upvalue named LUA_ENV +*/ +static void mainfunc (LexState *ls, FuncState *fs) { + BlockCnt bl; + expdesc v; + open_func(ls, fs, &bl); + fs->f->is_vararg = 1; /* main function is always vararg */ + init_exp(&v, VLOCAL, 0); /* create and... */ + newupvalue(fs, ls->envn, &v); /* ...set environment upvalue */ + luaX_next(ls); /* read first token */ + statlist(ls); /* parse main body */ + check(ls, TK_EOS); + close_func(ls); +} + + +Closure *luaY_parser (lua_State *L, ZIO *z, Mbuffer *buff, + Dyndata *dyd, const char *name, int firstchar) { LexState lexstate; FuncState funcstate; - BlockCnt bl; - TString *tname = luaS_new(L, name); - setsvalue2s(L, L->top, tname); /* push name to protect it */ + Closure *cl = luaF_newLclosure(L, 1); /* create main closure */ + /* anchor closure (to avoid being collected) */ + setclLvalue(L, L->top, cl); incr_top(L); + funcstate.f = cl->l.p = luaF_newproto(L); + funcstate.f->source = luaS_new(L, name); /* create and anchor TString */ lexstate.buff = buff; lexstate.dyd = dyd; dyd->actvar.n = dyd->gt.n = dyd->label.n = 0; - luaX_setinput(L, &lexstate, z, tname, firstchar); - open_mainfunc(&lexstate, &funcstate, &bl); - luaX_next(&lexstate); /* read first token */ - statlist(&lexstate); /* main body */ - check(&lexstate, TK_EOS); - close_func(&lexstate); - L->top--; /* pop name */ + luaX_setinput(L, &lexstate, z, funcstate.f->source, firstchar); + mainfunc(&lexstate, &funcstate); lua_assert(!funcstate.prev && funcstate.nups == 1 && !lexstate.fs); /* all scopes should be correctly finished */ lua_assert(dyd->actvar.n == 0 && dyd->gt.n == 0 && dyd->label.n == 0); - return funcstate.f; + return cl; /* it's on the stack too */ } diff --git a/depends/lua/src/lstate.c b/depends/lua/src/lstate.c index 6e2801c49..3c00c2855 100644 --- a/depends/lua/src/lstate.c +++ b/depends/lua/src/lstate.c @@ -1,11 +1,12 @@ /* -** $Id: lstate.c,v 2.92 2011/10/03 17:54:25 roberto Exp $ +** $Id: lstate.c,v 2.98 2012/05/30 12:33:44 roberto Exp $ ** Global State ** See Copyright Notice in lua.h */ #include +#include #define lstate_c #define LUA_CORE @@ -38,7 +39,18 @@ #endif -#define MEMERRMSG "not enough memory" +#define MEMERRMSG "not enough memory" + + +/* +** a macro to help the creation of a unique random seed when a state is +** created; the seed is used to randomize hashes. +*/ +#if !defined(luai_makeseed) +#include +#define luai_makeseed() cast(size_t, time(NULL)) +#endif + /* @@ -65,6 +77,28 @@ typedef struct LG { #define fromstate(L) (cast(LX *, cast(lu_byte *, (L)) - offsetof(LX, l))) +/* +** Compute an initial seed as random as possible. In ANSI, rely on +** Address Space Layout Randomization (if present) to increase +** randomness.. +*/ +#define addbuff(b,p,e) \ + { size_t t = cast(size_t, e); \ + memcpy(buff + p, &t, sizeof(t)); p += sizeof(t); } + +static unsigned int makeseed (lua_State *L) { + char buff[4 * sizeof(size_t)]; + unsigned int h = luai_makeseed(); + int p = 0; + addbuff(buff, p, L); /* heap variable */ + addbuff(buff, p, &h); /* local variable */ + addbuff(buff, p, luaO_nilobject); /* global variable */ + addbuff(buff, p, &lua_newstate); /* public function */ + lua_assert(p == sizeof(buff)); + return luaS_hash(buff, p, h); +} + + /* ** set GCdebt to a new value keeping the value (totalbytes + GCdebt) ** invariant @@ -242,10 +276,11 @@ LUA_API lua_State *lua_newstate (lua_Alloc f, void *ud) { g->frealloc = f; g->ud = ud; g->mainthread = L; + g->seed = makeseed(L); g->uvhead.u.l.prev = &g->uvhead; g->uvhead.u.l.next = &g->uvhead; g->gcrunning = 0; /* no GC while building state */ - g->lastmajormem = 0; + g->GCestimate = 0; g->strt.size = 0; g->strt.nuse = 0; g->strt.hash = NULL; @@ -257,6 +292,7 @@ LUA_API lua_State *lua_newstate (lua_Alloc f, void *ud) { g->allgc = NULL; g->finobj = NULL; g->tobefnz = NULL; + g->sweepgc = g->sweepfin = NULL; g->gray = g->grayagain = NULL; g->weak = g->ephemeron = g->allweak = NULL; g->totalbytes = sizeof(LG); diff --git a/depends/lua/src/lstring.c b/depends/lua/src/lstring.c index adec415e7..8b959f19d 100644 --- a/depends/lua/src/lstring.c +++ b/depends/lua/src/lstring.c @@ -1,5 +1,5 @@ /* -** $Id: lstring.c,v 2.19 2011/05/03 16:01:57 roberto Exp $ +** $Id: lstring.c,v 2.24 2012/05/11 14:14:42 roberto Exp $ ** String table (keeps all strings handled by Lua) ** See Copyright Notice in lua.h */ @@ -18,7 +18,49 @@ #include "lstring.h" +/* +** Lua will use at most ~(2^LUAI_HASHLIMIT) bytes from a string to +** compute its hash +*/ +#if !defined(LUAI_HASHLIMIT) +#define LUAI_HASHLIMIT 5 +#endif + + +/* +** equality for long strings +*/ +int luaS_eqlngstr (TString *a, TString *b) { + size_t len = a->tsv.len; + lua_assert(a->tsv.tt == LUA_TLNGSTR && b->tsv.tt == LUA_TLNGSTR); + return (a == b) || /* same instance or... */ + ((len == b->tsv.len) && /* equal length and ... */ + (memcmp(getstr(a), getstr(b), len) == 0)); /* equal contents */ +} + + +/* +** equality for strings +*/ +int luaS_eqstr (TString *a, TString *b) { + return (a->tsv.tt == b->tsv.tt) && + (a->tsv.tt == LUA_TSHRSTR ? eqshrstr(a, b) : luaS_eqlngstr(a, b)); +} + + +unsigned int luaS_hash (const char *str, size_t l, unsigned int seed) { + unsigned int h = seed ^ l; + size_t l1; + size_t step = (l >> LUAI_HASHLIMIT) + 1; + for (l1 = l; l1 >= step; l1 -= step) + h = h ^ ((h<<5) + (h>>2) + cast_byte(str[l1 - 1])); + return h; +} + +/* +** resizes the string table +*/ void luaS_resize (lua_State *L, int newsize) { int i; stringtable *tb = &G(L)->strt; @@ -50,37 +92,49 @@ void luaS_resize (lua_State *L, int newsize) { } -static TString *newlstr (lua_State *L, const char *str, size_t l, - unsigned int h) { - size_t totalsize; /* total size of TString object */ - GCObject **list; /* (pointer to) list where it will be inserted */ +/* +** creates a new string object +*/ +static TString *createstrobj (lua_State *L, const char *str, size_t l, + int tag, unsigned int h, GCObject **list) { TString *ts; - stringtable *tb = &G(L)->strt; - if (l+1 > (MAX_SIZET - sizeof(TString))/sizeof(char)) - luaM_toobig(L); - if (tb->nuse >= cast(lu_int32, tb->size) && tb->size <= MAX_INT/2) - luaS_resize(L, tb->size*2); /* too crowded */ + size_t totalsize; /* total size of TString object */ totalsize = sizeof(TString) + ((l + 1) * sizeof(char)); - list = &tb->hash[lmod(h, tb->size)]; - ts = &luaC_newobj(L, LUA_TSTRING, totalsize, list, 0)->ts; + ts = &luaC_newobj(L, tag, totalsize, list, 0)->ts; ts->tsv.len = l; ts->tsv.hash = h; - ts->tsv.reserved = 0; + ts->tsv.extra = 0; memcpy(ts+1, str, l*sizeof(char)); ((char *)(ts+1))[l] = '\0'; /* ending 0 */ - tb->nuse++; return ts; } -TString *luaS_newlstr (lua_State *L, const char *str, size_t l) { +/* +** creates a new short string, inserting it into string table +*/ +static TString *newshrstr (lua_State *L, const char *str, size_t l, + unsigned int h) { + GCObject **list; /* (pointer to) list where it will be inserted */ + stringtable *tb = &G(L)->strt; + TString *s; + if (tb->nuse >= cast(lu_int32, tb->size) && tb->size <= MAX_INT/2) + luaS_resize(L, tb->size*2); /* too crowded */ + list = &tb->hash[lmod(h, tb->size)]; + s = createstrobj(L, str, l, LUA_TSHRSTR, h, list); + tb->nuse++; + return s; +} + + +/* +** checks whether short string exists and reuses it or creates a new one +*/ +static TString *internshrstr (lua_State *L, const char *str, size_t l) { GCObject *o; - unsigned int h = cast(unsigned int, l); /* seed */ - size_t step = (l>>5)+1; /* if string is too long, don't hash all its chars */ - size_t l1; - for (l1=l; l1>=step; l1-=step) /* compute hash */ - h = h ^ ((h<<5)+(h>>2)+cast(unsigned char, str[l1-1])); - for (o = G(L)->strt.hash[lmod(h, G(L)->strt.size)]; + global_State *g = G(L); + unsigned int h = luaS_hash(str, l, g->seed); + for (o = g->strt.hash[lmod(h, g->strt.size)]; o != NULL; o = gch(o)->next) { TString *ts = rawgco2ts(o); @@ -92,10 +146,27 @@ TString *luaS_newlstr (lua_State *L, const char *str, size_t l) { return ts; } } - return newlstr(L, str, l, h); /* not found; create a new string */ + return newshrstr(L, str, l, h); /* not found; create a new string */ +} + + +/* +** new string (with explicit length) +*/ +TString *luaS_newlstr (lua_State *L, const char *str, size_t l) { + if (l <= LUAI_MAXSHORTLEN) /* short string? */ + return internshrstr(L, str, l); + else { + if (l + 1 > (MAX_SIZET - sizeof(TString))/sizeof(char)) + luaM_toobig(L); + return createstrobj(L, str, l, LUA_TLNGSTR, G(L)->seed, NULL); + } } +/* +** new zero-terminated string +*/ TString *luaS_new (lua_State *L, const char *str) { return luaS_newlstr(L, str, strlen(str)); } diff --git a/depends/lua/src/lstrlib.c b/depends/lua/src/lstrlib.c index f5d61fd85..e13098bb6 100644 --- a/depends/lua/src/lstrlib.c +++ b/depends/lua/src/lstrlib.c @@ -1,5 +1,5 @@ /* -** $Id: lstrlib.c,v 1.173 2011/11/30 18:24:56 roberto Exp $ +** $Id: lstrlib.c,v 1.176 2012/05/23 15:37:09 roberto Exp $ ** Standard library for string operations and pattern-matching ** See Copyright Notice in lua.h */ @@ -30,7 +30,7 @@ /* macro to `unsign' a character */ -#define uchar(c) ((unsigned char)(c)) +#define uchar(c) ((unsigned char)(c)) @@ -119,7 +119,9 @@ static int str_rep (lua_State *L) { char *p = luaL_buffinitsize(L, &b, totallen); while (n-- > 1) { /* first n-1 copies (followed by separator) */ memcpy(p, s, l * sizeof(char)); p += l; - memcpy(p, sep, lsep * sizeof(char)); p += lsep; + if (lsep > 0) { /* avoid empty 'memcpy' (may be expensive) */ + memcpy(p, sep, lsep * sizeof(char)); p += lsep; + } } memcpy(p, s, l * sizeof(char)); /* last copy (not followed by separator) */ luaL_pushresultsize(&b, totallen); @@ -745,20 +747,17 @@ static int str_gsub (lua_State *L) { #if !defined(LUA_INTFRMLEN) /* { */ #if defined(LUA_USE_LONGLONG) -#define LUA_INTFRMLEN "ll" -#define LUA_INTFRM_T long long +#define LUA_INTFRMLEN "ll" +#define LUA_INTFRM_T long long #else -#define LUA_INTFRMLEN "l" -#define LUA_INTFRM_T long +#define LUA_INTFRMLEN "l" +#define LUA_INTFRM_T long #endif #endif /* } */ -#define MAX_UINTFRM ((lua_Number)(~(unsigned LUA_INTFRM_T)0)) -#define MAX_INTFRM ((lua_Number)((~(unsigned LUA_INTFRM_T)0)/2)) -#define MIN_INTFRM (-(lua_Number)((~(unsigned LUA_INTFRM_T)0)/2) - 1) /* ** LUA_FLTFRMLEN is the length modifier for float conversions in @@ -767,8 +766,8 @@ static int str_gsub (lua_State *L) { */ #if !defined(LUA_FLTFRMLEN) -#define LUA_FLTFRMLEN "" -#define LUA_FLTFRM_T double +#define LUA_FLTFRMLEN "" +#define LUA_FLTFRM_T double #endif @@ -870,18 +869,22 @@ static int str_format (lua_State *L) { } case 'd': case 'i': { lua_Number n = luaL_checknumber(L, arg); - luaL_argcheck(L, (MIN_INTFRM - 1) < n && n < (MAX_INTFRM + 1), arg, + LUA_INTFRM_T ni = (LUA_INTFRM_T)n; + lua_Number diff = n - (lua_Number)ni; + luaL_argcheck(L, -1 < diff && diff < 1, arg, "not a number in proper range"); addlenmod(form, LUA_INTFRMLEN); - nb = sprintf(buff, form, (LUA_INTFRM_T)n); + nb = sprintf(buff, form, ni); break; } case 'o': case 'u': case 'x': case 'X': { lua_Number n = luaL_checknumber(L, arg); - luaL_argcheck(L, 0 <= n && n < (MAX_UINTFRM + 1), arg, + unsigned LUA_INTFRM_T ni = (unsigned LUA_INTFRM_T)n; + lua_Number diff = n - (lua_Number)ni; + luaL_argcheck(L, -1 < diff && diff < 1, arg, "not a non-negative number in proper range"); addlenmod(form, LUA_INTFRMLEN); - nb = sprintf(buff, form, (unsigned LUA_INTFRM_T)n); + nb = sprintf(buff, form, ni); break; } case 'e': case 'E': case 'f': diff --git a/depends/lua/src/ltable.c b/depends/lua/src/ltable.c index 9581add91..ffa5ecb36 100644 --- a/depends/lua/src/ltable.c +++ b/depends/lua/src/ltable.c @@ -1,5 +1,5 @@ /* -** $Id: ltable.c,v 2.67 2011/11/30 12:41:45 roberto Exp $ +** $Id: ltable.c,v 2.71 2012/05/23 15:37:09 roberto Exp $ ** Lua tables (hash) ** See Copyright Notice in lua.h */ @@ -48,10 +48,10 @@ #define MAXASIZE (1 << MAXBITS) -#define hashpow2(t,n) (gnode(t, lmod((n), sizenode(t)))) +#define hashpow2(t,n) (gnode(t, lmod((n), sizenode(t)))) -#define hashstr(t,str) hashpow2(t, (str)->tsv.hash) -#define hashboolean(t,p) hashpow2(t, p) +#define hashstr(t,str) hashpow2(t, (str)->tsv.hash) +#define hashboolean(t,p) hashpow2(t, p) /* @@ -98,7 +98,15 @@ static Node *mainposition (const Table *t, const TValue *key) { switch (ttype(key)) { case LUA_TNUMBER: return hashnum(t, nvalue(key)); - case LUA_TSTRING: + case LUA_TLNGSTR: { + TString *s = rawtsvalue(key); + if (s->tsv.extra == 0) { /* no hash? */ + s->tsv.hash = luaS_hash(getstr(s), s->tsv.len, s->tsv.hash); + s->tsv.extra = 1; /* now it has its hash */ + } + return hashstr(t, rawtsvalue(key)); + } + case LUA_TSHRSTR: return hashstr(t, rawtsvalue(key)); case LUA_TBOOLEAN: return hashboolean(t, bvalue(key)); @@ -453,12 +461,13 @@ const TValue *luaH_getint (Table *t, int key) { /* -** search function for strings +** search function for short strings */ const TValue *luaH_getstr (Table *t, TString *key) { Node *n = hashstr(t, key); + lua_assert(key->tsv.tt == LUA_TSHRSTR); do { /* check whether `key' is somewhere in the chain */ - if (ttisstring(gkey(n)) && eqstr(rawtsvalue(gkey(n)), key)) + if (ttisshrstring(gkey(n)) && eqshrstr(rawtsvalue(gkey(n)), key)) return gval(n); /* that's it */ else n = gnext(n); } while (n); @@ -470,9 +479,9 @@ const TValue *luaH_getstr (Table *t, TString *key) { ** main search function */ const TValue *luaH_get (Table *t, const TValue *key) { - switch (ttypenv(key)) { + switch (ttype(key)) { case LUA_TNIL: return luaO_nilobject; - case LUA_TSTRING: return luaH_getstr(t, rawtsvalue(key)); + case LUA_TSHRSTR: return luaH_getstr(t, rawtsvalue(key)); case LUA_TNUMBER: { int k; lua_Number n = nvalue(key); diff --git a/depends/lua/src/lua.c b/depends/lua/src/lua.c index e20ab86d2..7614c7030 100644 --- a/depends/lua/src/lua.c +++ b/depends/lua/src/lua.c @@ -1,5 +1,5 @@ /* -** $Id: lua.c,v 1.203 2011/12/12 16:34:03 roberto Exp $ +** $Id: lua.c,v 1.205 2012/05/23 15:37:09 roberto Exp $ ** Lua stand-alone interpreter ** See Copyright Notice in lua.h */ @@ -45,13 +45,13 @@ */ #if defined(LUA_USE_ISATTY) #include -#define lua_stdin_is_tty() isatty(0) +#define lua_stdin_is_tty() isatty(0) #elif defined(LUA_WIN) #include #include -#define lua_stdin_is_tty() _isatty(_fileno(stdin)) +#define lua_stdin_is_tty() _isatty(_fileno(stdin)) #else -#define lua_stdin_is_tty() 1 /* assume stdin is a tty */ +#define lua_stdin_is_tty() 1 /* assume stdin is a tty */ #endif @@ -66,19 +66,19 @@ #include #include #include -#define lua_readline(L,b,p) ((void)L, ((b)=readline(p)) != NULL) +#define lua_readline(L,b,p) ((void)L, ((b)=readline(p)) != NULL) #define lua_saveline(L,idx) \ if (lua_rawlen(L,idx) > 0) /* non-empty line? */ \ add_history(lua_tostring(L, idx)); /* add it to history */ -#define lua_freeline(L,b) ((void)L, free(b)) +#define lua_freeline(L,b) ((void)L, free(b)) #elif !defined(lua_readline) -#define lua_readline(L,b,p) \ +#define lua_readline(L,b,p) \ ((void)L, fputs(p, stdout), fflush(stdout), /* show prompt */ \ fgets(b, LUA_MAXINPUT, stdin) != NULL) /* get line */ -#define lua_saveline(L,idx) { (void)L; (void)idx; } -#define lua_freeline(L,b) { (void)L; (void)b; } +#define lua_saveline(L,idx) { (void)L; (void)idx; } +#define lua_freeline(L,b) { (void)L; (void)b; } #endif @@ -223,16 +223,11 @@ static int dostring (lua_State *L, const char *s, const char *name) { static int dolibrary (lua_State *L, const char *name) { int status; - lua_pushglobaltable(L); - lua_getfield(L, -1, "require"); + lua_getglobal(L, "require"); lua_pushstring(L, name); - status = docall(L, 1, 1); - if (status == LUA_OK) { - lua_setfield(L, -2, name); /* global[name] = require return */ - lua_pop(L, 1); /* remove global table */ - } - else - lua_remove(L, -2); /* remove global table (below error msg.) */ + status = docall(L, 1, 1); /* call 'require(name)' */ + if (status == LUA_OK) + lua_setglobal(L, name); /* global[name] = require return */ return report(L, status); } diff --git a/depends/lua/src/lundump.c b/depends/lua/src/lundump.c index 80c7aa39b..54de011a4 100644 --- a/depends/lua/src/lundump.c +++ b/depends/lua/src/lundump.c @@ -1,5 +1,5 @@ /* -** $Id: lundump.c,v 1.71 2011/12/07 10:39:12 lhf Exp $ +** $Id: lundump.c,v 2.22 2012/05/08 13:53:33 roberto Exp $ ** load precompiled Lua chunks ** See Copyright Notice in lua.h */ @@ -27,7 +27,7 @@ typedef struct { const char* name; } LoadState; -static void error(LoadState* S, const char* why) +static l_noret error(LoadState* S, const char* why) { luaO_pushfstring(S->L,"%s: %s precompiled chunk",S->name,why); luaD_throw(S->L,LUA_ERRSYNTAX); @@ -39,7 +39,7 @@ static void error(LoadState* S, const char* why) #define LoadVector(S,b,n,size) LoadMem(S,b,n,size) #if !defined(luai_verifycode) -#define luai_verifycode(L,b,f) (f) +#define luai_verifycode(L,b,f) /* empty */ #endif static void LoadBlock(LoadState* S, void* b, size_t size) @@ -91,7 +91,7 @@ static void LoadCode(LoadState* S, Proto* f) LoadVector(S,f->code,n,sizeof(Instruction)); } -static Proto* LoadFunction(LoadState* S); +static void LoadFunction(LoadState* S, Proto* f); static void LoadConstants(LoadState* S, Proto* f) { @@ -118,13 +118,18 @@ static void LoadConstants(LoadState* S, Proto* f) case LUA_TSTRING: setsvalue2n(S->L,o,LoadString(S)); break; + default: lua_assert(0); } } n=LoadInt(S); f->p=luaM_newvector(S->L,n,Proto*); f->sizep=n; for (i=0; ip[i]=NULL; - for (i=0; ip[i]=LoadFunction(S); + for (i=0; ip[i]=luaF_newproto(S->L); + LoadFunction(S,f->p[i]); + } } static void LoadUpvalues(LoadState* S, Proto* f) @@ -163,10 +168,8 @@ static void LoadDebug(LoadState* S, Proto* f) for (i=0; iupvalues[i].name=LoadString(S); } -static Proto* LoadFunction(LoadState* S) +static void LoadFunction(LoadState* S, Proto* f) { - Proto* f=luaF_newproto(S->L); - setptvalue2s(S->L,S->L->top,f); incr_top(S->L); f->linedefined=LoadInt(S); f->lastlinedefined=LoadInt(S); f->numparams=LoadByte(S); @@ -176,8 +179,6 @@ static Proto* LoadFunction(LoadState* S) LoadConstants(S,f); LoadUpvalues(S,f); LoadDebug(S,f); - S->L->top--; - return f; } /* the code below must be consistent with the code in luaU_header */ @@ -202,9 +203,10 @@ static void LoadHeader(LoadState* S) /* ** load precompiled chunk */ -Proto* luaU_undump (lua_State* L, ZIO* Z, Mbuffer* buff, const char* name) +Closure* luaU_undump (lua_State* L, ZIO* Z, Mbuffer* buff, const char* name) { LoadState S; + Closure* cl; if (*name=='@' || *name=='=') S.name=name+1; else if (*name==LUA_SIGNATURE[0]) @@ -215,7 +217,19 @@ Proto* luaU_undump (lua_State* L, ZIO* Z, Mbuffer* buff, const char* name) S.Z=Z; S.b=buff; LoadHeader(&S); - return luai_verifycode(L,buff,LoadFunction(&S)); + cl=luaF_newLclosure(L,1); + setclLvalue(L,L->top,cl); incr_top(L); + cl->l.p=luaF_newproto(L); + LoadFunction(&S,cl->l.p); + if (cl->l.p->sizeupvalues != 1) + { + Proto* p=cl->l.p; + cl=luaF_newLclosure(L,cl->l.p->sizeupvalues); + cl->l.p=p; + setclLvalue(L,L->top-1,cl); + } + luai_verifycode(L,buff,cl->l.p); + return cl; } #define MYINT(s) (s[0]-'0') diff --git a/depends/lua/src/lvm.c b/depends/lua/src/lvm.c index 694971b11..b77eac26d 100644 --- a/depends/lua/src/lvm.c +++ b/depends/lua/src/lvm.c @@ -1,5 +1,5 @@ /* -** $Id: lvm.c,v 2.147 2011/12/07 14:43:55 roberto Exp $ +** $Id: lvm.c,v 2.152 2012/06/08 15:14:04 roberto Exp $ ** Lua virtual machine ** See Copyright Notice in lua.h */ @@ -60,10 +60,15 @@ int luaV_tostring (lua_State *L, StkId obj) { static void traceexec (lua_State *L) { CallInfo *ci = L->ci; lu_byte mask = L->hookmask; - if ((mask & LUA_MASKCOUNT) && L->hookcount == 0) { - resethookcount(L); - luaD_hook(L, LUA_HOOKCOUNT, -1); + int counthook = ((mask & LUA_MASKCOUNT) && L->hookcount == 0); + if (counthook) + resethookcount(L); /* reset count */ + if (ci->callstatus & CIST_HOOKYIELD) { /* called hook last time? */ + ci->callstatus &= ~CIST_HOOKYIELD; /* erase mark */ + return; /* do not call hook again (VM yielded, so it did not move) */ } + if (counthook) + luaD_hook(L, LUA_HOOKCOUNT, -1); /* call count hook */ if (mask & LUA_MASKLINE) { Proto *p = ci_func(ci)->p; int npc = pcRel(ci->u.l.savedpc, p); @@ -71,11 +76,15 @@ static void traceexec (lua_State *L) { if (npc == 0 || /* call linehook when enter a new function, */ ci->u.l.savedpc <= L->oldpc || /* when jump back (loop), or when */ newline != getfuncline(p, pcRel(L->oldpc, p))) /* enter a new line */ - luaD_hook(L, LUA_HOOKLINE, newline); + luaD_hook(L, LUA_HOOKLINE, newline); /* call line hook */ } L->oldpc = ci->u.l.savedpc; if (L->status == LUA_YIELD) { /* did hook yield? */ + if (counthook) + L->hookcount = 1; /* undo decrement to zero */ ci->u.l.savedpc--; /* undo increment (resume will increment it again) */ + ci->callstatus |= CIST_HOOKYIELD; /* mark that it yieled */ + ci->func = L->top - 1; /* protect stack below results */ luaD_throw(L, LUA_YIELD); } } @@ -258,7 +267,8 @@ int luaV_equalobj_ (lua_State *L, const TValue *t1, const TValue *t2) { case LUA_TBOOLEAN: return bvalue(t1) == bvalue(t2); /* true must be 1 !! */ case LUA_TLIGHTUSERDATA: return pvalue(t1) == pvalue(t2); case LUA_TLCF: return fvalue(t1) == fvalue(t2); - case LUA_TSTRING: return eqstr(rawtsvalue(t1), rawtsvalue(t2)); + case LUA_TSHRSTR: return eqshrstr(rawtsvalue(t1), rawtsvalue(t2)); + case LUA_TLNGSTR: return luaS_eqlngstr(rawtsvalue(t1), rawtsvalue(t2)); case LUA_TUSERDATA: { if (uvalue(t1) == uvalue(t2)) return 1; else if (L == NULL) return 0; @@ -293,7 +303,7 @@ void luaV_concat (lua_State *L, int total) { else if (tsvalue(top-1)->len == 0) /* second operand is empty? */ (void)tostring(L, top - 2); /* result is first operand */ else if (ttisstring(top-2) && tsvalue(top-2)->len == 0) { - setsvalue2s(L, top-2, rawtsvalue(top-1)); /* result is second op. */ + setobjs2s(L, top - 2, top - 1); /* result is second op. */ } else { /* at least two non-empty string values; get as many as possible */ @@ -394,7 +404,8 @@ static void pushclosure (lua_State *L, Proto *p, UpVal **encup, StkId base, int nup = p->sizeupvalues; Upvaldesc *uv = p->upvalues; int i; - Closure *ncl = luaF_newLclosure(L, p); + Closure *ncl = luaF_newLclosure(L, nup); + ncl->l.p = p; setclLvalue(L, ra, ncl); /* anchor new closure in stack */ for (i = 0; i < nup; i++) { /* fill in its upvalues */ if (uv[i].instack) /* upvalue refers to local variable? */ @@ -500,7 +511,11 @@ void luaV_finishOp (lua_State *L) { #define Protect(x) { {x;}; base = ci->u.l.base; } -#define checkGC(L,c) Protect(luaC_condGC(L, c); luai_threadyield(L);) +#define checkGC(L,c) \ + Protect( luaC_condGC(L,{L->top = (c); /* limit of live values */ \ + luaC_step(L); \ + L->top = ci->top;}) /* restore top */ \ + luai_threadyield(L); ) #define arith_op(op,tm) { \ @@ -593,11 +608,7 @@ void luaV_execute (lua_State *L) { sethvalue(L, ra, t); if (b != 0 || c != 0) luaH_resize(L, t, luaO_fb2int(b), luaO_fb2int(c)); - checkGC(L, - L->top = ra + 1; /* limit of live values */ - luaC_step(L); - L->top = ci->top; /* restore top */ - ) + checkGC(L, ra + 1); ) vmcase(OP_SELF, StkId rb = RB(i); @@ -649,10 +660,7 @@ void luaV_execute (lua_State *L) { ra = RA(i); /* 'luav_concat' may invoke TMs and move the stack */ rb = b + base; setobjs2s(L, ra, rb); - checkGC(L, - L->top = (ra >= rb ? ra + 1 : rb); /* limit of live values */ - luaC_step(L); - ) + checkGC(L, (ra >= rb ? ra + 1 : rb)); L->top = ci->top; /* restore top */ ) vmcase(OP_JMP, @@ -830,11 +838,7 @@ void luaV_execute (lua_State *L) { pushclosure(L, p, cl->upvals, base, ra); /* create a new one */ else setclLvalue(L, ra, ncl); /* push cashed closure */ - checkGC(L, - L->top = ra + 1; /* limit of live values */ - luaC_step(L); - L->top = ci->top; /* restore top */ - ) + checkGC(L, ra + 1); ) vmcase(OP_VARARG, int b = GETARG_B(i) - 1; diff --git a/depends/lua/src/lzio.c b/depends/lua/src/lzio.c index 354f94e7c..8b77054e0 100644 --- a/depends/lua/src/lzio.c +++ b/depends/lua/src/lzio.c @@ -1,6 +1,6 @@ /* -** $Id: lzio.c,v 1.34 2011/07/15 12:35:32 roberto Exp $ -** a generic input stream interface +** $Id: lzio.c,v 1.35 2012/05/14 13:34:18 roberto Exp $ +** Buffered streams ** See Copyright Notice in lua.h */ From eeaae916edc28baa065e277acde69b0674fd3fd2 Mon Sep 17 00:00:00 2001 From: Alexander Gavrilov Date: Sun, 17 Jun 2012 16:01:08 +0400 Subject: [PATCH 14/28] Reapply dfhack-specific lua build configuration. --- depends/lua/include/luaconf.h | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/depends/lua/include/luaconf.h b/depends/lua/include/luaconf.h index e4335df90..766752ff8 100644 --- a/depends/lua/include/luaconf.h +++ b/depends/lua/include/luaconf.h @@ -89,25 +89,21 @@ ** In Windows, any exclamation mark ('!') in the path is replaced by the ** path of the directory of the executable file of the current process. */ -#define LUA_LDIR "!\\lua\\" +#define LUA_LDIR "!\\hack\\lua\\" #define LUA_CDIR "!\\" #define LUA_PATH_DEFAULT \ - LUA_LDIR"?.lua;" LUA_LDIR"?\\init.lua;" \ - LUA_CDIR"?.lua;" LUA_CDIR"?\\init.lua;" ".\\?.lua" + LUA_LDIR"?.lua;" LUA_LDIR"?\\init.lua;" ".\\?.lua" #define LUA_CPATH_DEFAULT \ - LUA_CDIR"?.dll;" LUA_CDIR"loadall.dll;" ".\\?.dll" + LUA_CDIR"?.dll;" ".\\?.dll" #else /* }{ */ -#define LUA_VDIR LUA_VERSION_MAJOR "." LUA_VERSION_MINOR "/" -#define LUA_ROOT "/usr/local/" -#define LUA_LDIR LUA_ROOT "share/lua/" LUA_VDIR -#define LUA_CDIR LUA_ROOT "lib/lua/" LUA_VDIR +#define LUA_LDIR "./hack/lua/" +#define LUA_CDIR "./hack/" #define LUA_PATH_DEFAULT \ - LUA_LDIR"?.lua;" LUA_LDIR"?/init.lua;" \ - LUA_CDIR"?.lua;" LUA_CDIR"?/init.lua;" "./?.lua" + LUA_LDIR"?.lua;" LUA_LDIR"?/init.lua;" "./?.lua" #define LUA_CPATH_DEFAULT \ - LUA_CDIR"?.so;" LUA_CDIR"loadall.so;" "./?.so" + LUA_CDIR"?.so;" "./?.so" #endif /* } */ @@ -142,11 +138,15 @@ */ #if defined(LUA_BUILD_AS_DLL) /* { */ +#if defined(_MSC_VER) #if defined(LUA_CORE) || defined(LUA_LIB) /* { */ #define LUA_API __declspec(dllexport) #else /* }{ */ #define LUA_API __declspec(dllimport) #endif /* } */ +#else +#define LUA_API __attribute__ ((visibility("default"))) +#endif #else /* }{ */ From 16ae6cb1fb06217e5b32f74ec72ce9fce1923b56 Mon Sep 17 00:00:00 2001 From: Alexander Gavrilov Date: Sun, 17 Jun 2012 15:48:32 +0400 Subject: [PATCH 15/28] Fix lua bug: 'peaceful' unwind of yielded pcall doesn't restore errfunc. I.e. if the pcall results in no error, but a yield happens inside it, the active errfunc is not restored to its previous value. --- depends/lua/src/ldo.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/depends/lua/src/ldo.c b/depends/lua/src/ldo.c index d18e33cd4..35d95ae4e 100644 --- a/depends/lua/src/ldo.c +++ b/depends/lua/src/ldo.c @@ -402,6 +402,9 @@ static void finishCcall (lua_State *L) { int n; lua_assert(ci->u.c.k != NULL); /* must have a continuation */ lua_assert(L->nny == 0); + /* finish 'lua_pcallk' */ + if (ci->callstatus & CIST_YPCALL) + L->errfunc = ci->u.c.old_errfunc; /* finish 'lua_callk' */ adjustresults(L, ci->nresults); /* call continuation function */ From dc6cb61979458d00aa5c9a8d1f635d35891ca6ee Mon Sep 17 00:00:00 2001 From: Alexander Gavrilov Date: Sun, 17 Jun 2012 18:44:59 +0400 Subject: [PATCH 16/28] Add more offset finders to the script. --- library/lua/memscan.lua | 98 +++++++---- library/lua/utils.lua | 6 +- scripts/devel/find-offsets.lua | 288 +++++++++++++++++++++++++++------ 3 files changed, 311 insertions(+), 81 deletions(-) diff --git a/library/lua/memscan.lua b/library/lua/memscan.lua index e5a0c6f72..95b9197b1 100644 --- a/library/lua/memscan.lua +++ b/library/lua/memscan.lua @@ -344,62 +344,41 @@ function DiffSearcher:idx2addr(idx) return self.area[self.type]:idx2addr(idx) end --- Menu search utility +-- Interactive search utility -function find_menu_cursor(searcher,prompt,data_type,choices,enum) +function DiffSearcher:find_interactive(prompt,data_type,condition_cb) enum = enum or {} -- Loop for restarting search from scratch while true do print('\n'..prompt) - searcher:begin_search(data_type) + self:begin_search(data_type) local found = false local ccursor = 0 -- Loop through choices while true do - local choice + print('') - -- Select the next value to search for - if type(choices) == 'function' then - print('') + local ok, value, delta = condition_cb(ccursor) - choice = choices(ccursor) - ccursor = ccursor + 1 - - if not choice then - break - end - else - choice = choices[ccursor+1] - ccursor = (ccursor+1) % #choices + ccursor = ccursor + 1 - local cname = enum[choice] or choice - if type(choice) == 'string' and type(cname) == 'number' then - choice, cname = cname, choice - end - if cname ~= choice then - cname = cname..' ('..choice..')' - end - - -- Ask the user to select it - print('\n Please select: '..cname) - if not utils.prompt_yes_no(' Continue?', true) then - break - end + if not ok then + break end -- Search for it in the memory - local cnt, set = searcher:advance_search(choice) + local cnt, set = self:advance_search(value, delta) if not cnt then dfhack.printerr(' Converged to zero candidates; probably a mistake somewhere.') break elseif set and cnt == 1 then -- To confirm, wait for two 1-candidate results in a row if found then - local addr = searcher:idx2addr(set[1]) + local addr = self:idx2addr(set[1]) print(string.format(' Confirmed address: %x\n', addr)) return addr, set[1] else @@ -416,4 +395,61 @@ function find_menu_cursor(searcher,prompt,data_type,choices,enum) end end +function DiffSearcher:find_menu_cursor(prompt,data_type,choices,enum) + enum = enum or {} + + return self:find_interactive( + prompt, data_type, + function(ccursor) + local choice + + -- Select the next value to search for + if type(choices) == 'function' then + choice = choices(ccursor) + + if not choice then + return false + end + else + choice = choices[(ccursor % #choices) + 1] + end + + -- Ask the user to select it + if enum ~= 'noprompt' then + local cname = enum[choice] or choice + if type(choice) == 'string' and type(cname) == 'number' then + choice, cname = cname, choice + end + if cname ~= choice then + cname = cname..' ('..choice..')' + end + + print(' Please select: '..cname) + if not utils.prompt_yes_no(' Continue?', true) then + return false + end + end + + return true, choice + end + ) +end + +function DiffSearcher:find_counter(prompt,data_type,delta,action_prompt) + delta = delta or 1 + + return self:find_interactive( + prompt, data_type, + function(ccursor) + if ccursor > 0 then + print(" "..(action_prompt or 'Please do the action.')) + end + if not utils.prompt_yes_no(' Continue?', true) then + return false + end + return true, nil, delta + end + ) +end + return _ENV diff --git a/library/lua/utils.lua b/library/lua/utils.lua index 4fac56ece..93ee840c4 100644 --- a/library/lua/utils.lua +++ b/library/lua/utils.lua @@ -367,9 +367,9 @@ function prompt_yes_no(msg,default) if default == nil then prompt = prompt..' (y/n): ' elseif default then - prompt = prompt..' (y/n/enter=y): ' + prompt = prompt..' (y/n)[y]: ' else - prompt = prompt..' (y/n/enter=n): ' + prompt = prompt..' (y/n)[n]: ' end while true do local rv = dfhack.lineedit(prompt) @@ -378,6 +378,8 @@ function prompt_yes_no(msg,default) return true elseif string.match(rv,'^[Nn]') then return false + elseif rv == 'abort' then + error('User abort in utils.prompt_yes_no()') elseif rv == '' and default ~= nil then return default end diff --git a/scripts/devel/find-offsets.lua b/scripts/devel/find-offsets.lua index ef16026e3..c9293b7b5 100644 --- a/scripts/devel/find-offsets.lua +++ b/scripts/devel/find-offsets.lua @@ -3,9 +3,19 @@ local utils = require 'utils' local ms = require 'memscan' -local scan_all = false local is_known = dfhack.internal.getAddress +local scan_all = false +local force_scan = {} + +for _,v in ipairs({...}) do + if v == 'all' then + scan_all = true + else + force_scan[v] = true + end +end + collectgarbage() print[[ @@ -48,6 +58,52 @@ local function validate_offset(name,validator,addr,tname,...) ms.found_offset(name,obj) end +local function exec_finder(finder, names) + local search = scan_all + if type(names) ~= 'table' then + names = { names } + end + for _,v in ipairs(names) do + if force_scan[v] or not is_known(v) then + search = true + end + end + if search then + if not dfhack.safecall(finder) then + if not utils.prompt_yes_no('Proceed with the rest of the script?') then + searcher:reset() + error('Quit') + end + end + else + print('Already known: '..table.concat(names,', ')) + end +end + +local ordinal_names = { + [0] = '1st entry', + [1] = '2nd entry' +} +setmetatable(ordinal_names, { + __index = function(self,idx) return (idx+1)..'th entry' end +}) + +local function list_index_choices(length_func) + return function(id) + if id > 0 then + local ok, len = pcall(length_func) + if not ok then + len = 5 + elseif len > 10 then + len = 10 + end + return id % len + else + return 0 + end + end +end + -- -- Cursor group -- @@ -90,19 +146,14 @@ local function find_cursor() return false end -if scan_all or not ( - is_known 'cursor' and is_known 'selection_rect' and - is_known 'gamemode' and is_known 'gametype' -) then - find_cursor() -end +exec_finder(find_cursor, { 'cursor', 'selection_rect', 'gamemode', 'gametype' }) -- -- Announcements -- local function find_announcements() - idx, addr = data.int32_t:find_one{ + local idx, addr = data.int32_t:find_one{ 25, 25, 31, 31, 24, 24, 40, 40, 40, 40, 40, 40, 40 } if idx then @@ -113,9 +164,7 @@ local function find_announcements() dfhack.printerr('Could not find announcements.') end -if scan_all or not is_known 'announcements' then - find_announcements() -end +exec_finder(find_announcements, 'announcements') -- -- d_init @@ -142,7 +191,7 @@ local function is_valid_d_init(di) end local function find_d_init() - idx, addr = data.int16_t:find_one{ + local idx, addr = data.int16_t:find_one{ 1,0, 2,0, 5,0, 25,0, -- path_cost 4,4, -- embark_rect 20,1000,1000,1000,1000 -- store_dist @@ -155,10 +204,30 @@ local function find_d_init() dfhack.printerr('Could not find d_init') end -if scan_all or not is_known 'd_init' then - find_d_init() +exec_finder(find_d_init, 'd_init') + +-- +-- gview +-- + +local function find_gview() + local vs_vtable = dfhack.internal.getVTable('viewscreenst') + if not vs_vtable then + dfhack.printerr('Cannot search for gview - no viewscreenst vtable.') + return + end + + local idx, addr = data.uint32_t:find_one{0, vs_vtable} + if idx then + ms.found_offset('gview', addr) + return + end + + dfhack.printerr('Could not find gview') end +exec_finder(find_gview, 'gview') + -- -- World -- @@ -184,10 +253,9 @@ local function is_valid_world(world) end local function find_world() - local addr = ms.find_menu_cursor( - searcher, [[ + local addr = searcher:find_menu_cursor([[ Searching for world. Please open the stockpile creation -menu, and follow instructions below:]], +menu, and select different types as instructed below:]], 'int32_t', { 'Corpses', 'Refuse', 'Stone', 'Wood', 'Gems', 'Bars', 'Cloth', 'Leather', 'Ammo', 'Coins' }, df.stockpile_category @@ -195,9 +263,7 @@ menu, and follow instructions below:]], validate_offset('world', is_valid_world, addr, df.world, 'selected_stockpile_type') end -if scan_all or not is_known 'world' then - find_world() -end +exec_finder(find_world, 'world') -- -- UI @@ -220,10 +286,9 @@ local function is_valid_ui(ui) end local function find_ui() - local addr = ms.find_menu_cursor( - searcher, [[ + local addr = searcher:find_menu_cursor([[ Searching for ui. Please open the designation -menu, and follow instructions below:]], +menu, and switch modes as instructed below:]], 'int16_t', { 'DesignateMine', 'DesignateChannel', 'DesignateRemoveRamps', 'DesignateUpStair', 'DesignateDownStair', 'DesignateUpDownStair', 'DesignateUpRamp', 'DesignateChopTrees' }, @@ -232,9 +297,7 @@ menu, and follow instructions below:]], validate_offset('ui', is_valid_ui, addr, df.ui, 'main', 'mode') end -if scan_all or not is_known 'ui' then - find_ui() -end +exec_finder(find_ui, 'ui') -- -- ui_sidebar_menus @@ -261,21 +324,19 @@ local function is_valid_ui_sidebar_menus(usm) end local function find_ui_sidebar_menus() - local addr = ms.find_menu_cursor( - searcher, [[ + local addr = searcher:find_menu_cursor([[ Searching for ui_sidebar_menus. Please open the add job ui of Mason, Craftsdwarfs, or Carpenters workshop, and select entries in the list:]], 'int32_t', - { 0, 1, 2, 3, 4, 5, 6 } + { 0, 1, 2, 3, 4, 5, 6 }, + ordinal_names ) validate_offset('ui_sidebar_menus', is_valid_ui_sidebar_menus, addr, df.ui_sidebar_menus, 'workshop_job', 'cursor') end -if scan_all or not is_known 'ui_sidebar_menus' then - find_ui_sidebar_menus() -end +exec_finder(find_ui_sidebar_menus, 'ui_sidebar_menus') -- -- ui_build_selector @@ -299,8 +360,7 @@ local function is_valid_ui_build_selector(ubs) end local function find_ui_build_selector() - local addr = ms.find_menu_cursor( - searcher, [[ + local addr = searcher:find_menu_cursor([[ Searching for ui_build_selector. Please start constructing a pressure plate, and enable creatures. Then change the min weight as requested, remembering that the ui truncates the @@ -312,9 +372,7 @@ number, so when it shows "Min (5000df", it means 50000:]], addr, df.ui_build_selector, 'plate_info', 'unit_min') end -if scan_all or not is_known 'ui_build_selector' then - find_ui_build_selector() -end +exec_finder(find_ui_build_selector, 'ui_build_selector') -- -- ui_selected_unit @@ -322,7 +380,7 @@ end local function find_ui_selected_unit() if not is_known 'world' then - dfhack.printerr('Cannot find ui_selected_unit: no world') + dfhack.printerr('Cannot search for ui_selected_unit: no world') return end @@ -330,30 +388,27 @@ local function find_ui_selected_unit() dfhack.units.setNickname(unit, i) end - local addr = ms.find_menu_cursor( - searcher, [[ + local addr = searcher:find_menu_cursor([[ Searching for ui_selected_unit. Please activate the 'v' mode, point it at units, and enter their numeric nickname into the prompts below:]], 'int32_t', function() return utils.prompt_input(' Enter index: ', utils.check_number) - end + end, + 'noprompt' ) ms.found_offset('ui_selected_unit', addr) end -if scan_all or not is_known 'ui_selected_unit' then - find_ui_selected_unit() -end +exec_finder(find_ui_selected_unit, 'ui_selected_unit') -- -- ui_unit_view_mode -- local function find_ui_unit_view_mode() - local addr = ms.find_menu_cursor( - searcher, [[ + local addr = searcher:find_menu_cursor([[ Searching for ui_unit_view_mode. Having selected a unit with 'v', switch the pages as requested:]], 'int32_t', @@ -363,6 +418,143 @@ with 'v', switch the pages as requested:]], ms.found_offset('ui_unit_view_mode', addr) end -if scan_all or not is_known 'ui_unit_view_mode' then - find_ui_unit_view_mode() +exec_finder(find_ui_unit_view_mode, 'ui_unit_view_mode') + +-- +-- ui_look_cursor +-- + +local function look_item_list_count() + return #df.global.ui_look_list.items +end + +local function find_ui_look_cursor() + local addr = searcher:find_menu_cursor([[ +Searching for ui_look_cursor. Please activate the 'k' +mode, find a tile with many items or units on the ground, +and select list entries as instructed:]], + 'int32_t', + list_index_choices(look_item_list_count), + ordinal_names + ) + ms.found_offset('ui_look_cursor', addr) +end + +exec_finder(find_ui_look_cursor, 'ui_look_cursor') + +-- +-- ui_building_item_cursor +-- + +local function building_item_list_count() + return #df.global.world.selected_building.contained_items +end + +local function find_ui_building_item_cursor() + local addr = searcher:find_menu_cursor([[ +Searching for ui_building_item_cursor. Please activate the 't' +mode, find a cluttered workshop, trade depot, or other building +with many contained items, and select as instructed:]], + 'int32_t', + list_index_choices(building_item_list_count), + ordinal_names + ) + ms.found_offset('ui_building_item_cursor', addr) +end + +exec_finder(find_ui_building_item_cursor, 'ui_building_item_cursor') + +-- +-- ui_workshop_in_add +-- + +local function find_ui_workshop_in_add() + local addr = searcher:find_menu_cursor([[ +Searching for ui_workshop_in_add. Please activate the 'q' +mode, find a workshop without jobs (or delete jobs), +and do as instructed:]], + 'int8_t', + { 1, 0 }, + { [1] = 'enter the add job menu', + [0] = 'add job, thus exiting the menu' } + ) + ms.found_offset('ui_workshop_in_add', addr) +end + +--exec_finder(find_ui_workshop_in_add, 'ui_workshop_in_add') + +-- +-- ui_workshop_job_cursor +-- + +local function workshop_job_list_count() + return #df.global.world.selected_building.jobs +end + +local function find_ui_workshop_job_cursor() + local addr = searcher:find_menu_cursor([[ +Searching for ui_workshop_job_cursor. Please activate the 'q' +mode, find a workshop with many jobs, and select as instructed:]], + 'int32_t', + list_index_choices(workshop_job_list_count), + ordinal_names + ) + ms.found_offset('ui_workshop_job_cursor', addr) +end + +exec_finder(find_ui_workshop_job_cursor, 'ui_workshop_job_cursor') + +-- +-- window_x +-- + +local function find_window_x() + local addr = searcher:find_counter([[ +Searching for window_x. Please exit to main dwarfmode menu, +scroll to the LEFT edge, then do as instructed:]], + 'int32_t', 10, + 'Please press Right to scroll right one step.' + ) + ms.found_offset('window_x', addr) +end + +exec_finder(find_window_x, 'window_x') + +-- +-- window_y +-- + +local function find_window_y() + local addr = searcher:find_counter([[ +Searching for window_y. Please exit to main dwarfmode menu, +scroll to the TOP edge, then do as instructed:]], + 'int32_t', 10, + 'Please press Down to scroll down one step.' + ) + ms.found_offset('window_y', addr) end + +exec_finder(find_window_y, 'window_y') + +-- +-- window_z +-- + +local function find_window_z() + local addr = searcher:find_counter([[ +Searching for window_z. Please exit to main dwarfmode menu, +scroll to ground level, then do as instructed:]], + 'int32_t', -1, + "Please press '>' to scroll 1 Z level down." + ) + ms.found_offset('window_z', addr) +end + +--exec_finder(find_window_z, 'window_z') + +-- +-- THE END +-- + +print('Done.') +searcher:reset() From a6bfc670389698b5d3e61e7322a5231450c420f6 Mon Sep 17 00:00:00 2001 From: Alexander Gavrilov Date: Sun, 17 Jun 2012 21:40:08 +0400 Subject: [PATCH 17/28] Find a tweak to make two search recipes work, and add two more. --- scripts/devel/find-offsets.lua | 69 +++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/scripts/devel/find-offsets.lua b/scripts/devel/find-offsets.lua index c9293b7b5..6b3c31576 100644 --- a/scripts/devel/find-offsets.lua +++ b/scripts/devel/find-offsets.lua @@ -5,15 +5,9 @@ local ms = require 'memscan' local is_known = dfhack.internal.getAddress -local scan_all = false local force_scan = {} - for _,v in ipairs({...}) do - if v == 'all' then - scan_all = true - else - force_scan[v] = true - end + force_scan[v] = true end collectgarbage() @@ -43,7 +37,7 @@ if not data then error('Could not find data segment') end -print('Data section: '..tostring(data)) +print('\nData section: '..tostring(data)) if data.size < 5000000 then error('Data segment too short.') end @@ -59,10 +53,10 @@ local function validate_offset(name,validator,addr,tname,...) end local function exec_finder(finder, names) - local search = scan_all if type(names) ~= 'table' then names = { names } end + local search = force_scan['all'] for _,v in ipairs(names) do if force_scan[v] or not is_known(v) then search = true @@ -472,7 +466,9 @@ local function find_ui_workshop_in_add() local addr = searcher:find_menu_cursor([[ Searching for ui_workshop_in_add. Please activate the 'q' mode, find a workshop without jobs (or delete jobs), -and do as instructed:]], +and do as instructed below. + +NOTE: After first 3 steps resize the game window.]], 'int8_t', { 1, 0 }, { [1] = 'enter the add job menu', @@ -481,7 +477,7 @@ and do as instructed:]], ms.found_offset('ui_workshop_in_add', addr) end ---exec_finder(find_ui_workshop_in_add, 'ui_workshop_in_add') +exec_finder(find_ui_workshop_in_add, 'ui_workshop_in_add') -- -- ui_workshop_job_cursor @@ -504,6 +500,49 @@ end exec_finder(find_ui_workshop_job_cursor, 'ui_workshop_job_cursor') +-- +-- ui_building_in_assign +-- + +local function find_ui_building_in_assign() + local addr = searcher:find_menu_cursor([[ +Searching for ui_building_in_assign. Please activate +the 'q' mode, select a room building (e.g. a bedroom) +and do as instructed below. + +NOTE: After first 3 steps resize the game window.]], + 'int8_t', + { 1, 0 }, + { [1] = 'enter the Assign owner menu', + [0] = 'press Esc to exit assign' } + ) + ms.found_offset('ui_building_in_assign', addr) +end + +exec_finder(find_ui_building_in_assign, 'ui_building_in_assign') + +-- +-- ui_building_in_resize +-- + +local function find_ui_building_in_resize() + local addr = searcher:find_menu_cursor([[ +Searching for ui_building_in_resize. Please activate +the 'q' mode, select a room building (e.g. a bedroom) +and do as instructed below. + +NOTE: After first 3 steps resize the game window.]], + 'int8_t', + { 1, 0 }, + { [1] = 'enter the Resize room mode', + [0] = 'press Esc to exit resize' } + ) + ms.found_offset('ui_building_in_resize', addr) +end + +exec_finder(find_ui_building_in_resize, 'ui_building_in_resize') + + -- -- window_x -- @@ -543,14 +582,16 @@ exec_finder(find_window_y, 'window_y') local function find_window_z() local addr = searcher:find_counter([[ Searching for window_z. Please exit to main dwarfmode menu, -scroll to ground level, then do as instructed:]], +scroll to ground level, then do as instructed below. + +NOTE: After first 3 steps resize the game window.]], 'int32_t', -1, - "Please press '>' to scroll 1 Z level down." + "Please press '>' to scroll one Z level down." ) ms.found_offset('window_z', addr) end ---exec_finder(find_window_z, 'window_z') +exec_finder(find_window_z, 'window_z') -- -- THE END From bd5b675fa5ce1c8561b9d670180ae80ef4a2a2b1 Mon Sep 17 00:00:00 2001 From: Alexander Gavrilov Date: Mon, 18 Jun 2012 21:11:54 +0400 Subject: [PATCH 18/28] Add ui_menu_width finder, tweak instructions, and add a case for win exe. --- library/lua/memscan.lua | 3 +- scripts/devel/find-offsets.lua | 118 ++++++++++++++++++--------------- 2 files changed, 68 insertions(+), 53 deletions(-) diff --git a/library/lua/memscan.lua b/library/lua/memscan.lua index 95b9197b1..4bd01c8f7 100644 --- a/library/lua/memscan.lua +++ b/library/lua/memscan.lua @@ -208,7 +208,8 @@ local function find_data_segment() end elseif mem.read and mem.write and (string.match(mem.name,'/dwarfort%.exe$') - or string.match(mem.name,'/Dwarf_Fortress$')) + or string.match(mem.name,'/Dwarf_Fortress$') + or string.match(mem.name,'Dwarf Fortress%.exe')) then data_start = mem.start_addr data_end = mem.end_addr diff --git a/scripts/devel/find-offsets.lua b/scripts/devel/find-offsets.lua index 6b3c31576..fddfbc948 100644 --- a/scripts/devel/find-offsets.lua +++ b/scripts/devel/find-offsets.lua @@ -20,10 +20,14 @@ MAKE IT RUN CORRECTLY if any data structures changed, thus possibly leading to CRASHES AND/OR PERMANENT SAVE CORRUPTION. -This script should be initially started immediately -after loading the game, WITHOUT first loading a world. -It expects vanilla game configuration, without any -custom tilesets or init file changes. +Finding the first few globals requires this script to be +started immediately after loading the game, WITHOUT +first loading a world. + +The script expects vanilla game configuration, without +any custom tilesets or init file changes. Never unpause +the game unless instructed. When done, quit the game +without saving using 'die'. ]] if not utils.prompt_yes_no('Proceed?') then @@ -140,8 +144,6 @@ local function find_cursor() return false end -exec_finder(find_cursor, { 'cursor', 'selection_rect', 'gamemode', 'gametype' }) - -- -- Announcements -- @@ -158,8 +160,6 @@ local function find_announcements() dfhack.printerr('Could not find announcements.') end -exec_finder(find_announcements, 'announcements') - -- -- d_init -- @@ -198,8 +198,6 @@ local function find_d_init() dfhack.printerr('Could not find d_init') end -exec_finder(find_d_init, 'd_init') - -- -- gview -- @@ -220,8 +218,6 @@ local function find_gview() dfhack.printerr('Could not find gview') end -exec_finder(find_gview, 'gview') - -- -- World -- @@ -257,8 +253,6 @@ menu, and select different types as instructed below:]], validate_offset('world', is_valid_world, addr, df.world, 'selected_stockpile_type') end -exec_finder(find_world, 'world') - -- -- UI -- @@ -291,8 +285,6 @@ menu, and switch modes as instructed below:]], validate_offset('ui', is_valid_ui, addr, df.ui, 'main', 'mode') end -exec_finder(find_ui, 'ui') - -- -- ui_sidebar_menus -- @@ -319,9 +311,9 @@ end local function find_ui_sidebar_menus() local addr = searcher:find_menu_cursor([[ -Searching for ui_sidebar_menus. Please open the add job -ui of Mason, Craftsdwarfs, or Carpenters workshop, and -select entries in the list:]], +Searching for ui_sidebar_menus. Please switch to 'q' mode, +select a Mason, Craftsdwarfs, or Carpenters workshop, open +the Add Job menu, and move the cursor within:]], 'int32_t', { 0, 1, 2, 3, 4, 5, 6 }, ordinal_names @@ -330,8 +322,6 @@ select entries in the list:]], addr, df.ui_sidebar_menus, 'workshop_job', 'cursor') end -exec_finder(find_ui_sidebar_menus, 'ui_sidebar_menus') - -- -- ui_build_selector -- @@ -366,7 +356,25 @@ number, so when it shows "Min (5000df", it means 50000:]], addr, df.ui_build_selector, 'plate_info', 'unit_min') end -exec_finder(find_ui_build_selector, 'ui_build_selector') +-- +-- ui_menu_width +-- + +local function find_ui_menu_width() + local addr = searcher:find_menu_cursor([[ +Searching for ui_menu_width. Please exit to the main +dwarfmode menu, then use Tab to do as instructed below:]], + 'int8_t', + { 2, 3, 1 }, + { [2] = 'switch to the most usual [mapmap][menu] layout', + [3] = 'hide the menu completely', + [1] = 'switch to the default [map][menu][map] layout' } + ) + ms.found_offset('ui_menu_width', addr) + + -- NOTE: Assume that the vars are adjacent, as always + ms.found_offset('ui_area_map_width', addr+1) +end -- -- ui_selected_unit @@ -395,8 +403,6 @@ into the prompts below:]], ms.found_offset('ui_selected_unit', addr) end -exec_finder(find_ui_selected_unit, 'ui_selected_unit') - -- -- ui_unit_view_mode -- @@ -412,8 +418,6 @@ with 'v', switch the pages as requested:]], ms.found_offset('ui_unit_view_mode', addr) end -exec_finder(find_ui_unit_view_mode, 'ui_unit_view_mode') - -- -- ui_look_cursor -- @@ -434,8 +438,6 @@ and select list entries as instructed:]], ms.found_offset('ui_look_cursor', addr) end -exec_finder(find_ui_look_cursor, 'ui_look_cursor') - -- -- ui_building_item_cursor -- @@ -456,8 +458,6 @@ with many contained items, and select as instructed:]], ms.found_offset('ui_building_item_cursor', addr) end -exec_finder(find_ui_building_item_cursor, 'ui_building_item_cursor') - -- -- ui_workshop_in_add -- @@ -468,7 +468,7 @@ Searching for ui_workshop_in_add. Please activate the 'q' mode, find a workshop without jobs (or delete jobs), and do as instructed below. -NOTE: After first 3 steps resize the game window.]], +NOTE: If not done after first 3-4 steps, resize the game window.]], 'int8_t', { 1, 0 }, { [1] = 'enter the add job menu', @@ -477,8 +477,6 @@ NOTE: After first 3 steps resize the game window.]], ms.found_offset('ui_workshop_in_add', addr) end -exec_finder(find_ui_workshop_in_add, 'ui_workshop_in_add') - -- -- ui_workshop_job_cursor -- @@ -498,8 +496,6 @@ mode, find a workshop with many jobs, and select as instructed:]], ms.found_offset('ui_workshop_job_cursor', addr) end -exec_finder(find_ui_workshop_job_cursor, 'ui_workshop_job_cursor') - -- -- ui_building_in_assign -- @@ -510,7 +506,7 @@ Searching for ui_building_in_assign. Please activate the 'q' mode, select a room building (e.g. a bedroom) and do as instructed below. -NOTE: After first 3 steps resize the game window.]], +NOTE: If not done after first 3-4 steps, resize the game window.]], 'int8_t', { 1, 0 }, { [1] = 'enter the Assign owner menu', @@ -519,8 +515,6 @@ NOTE: After first 3 steps resize the game window.]], ms.found_offset('ui_building_in_assign', addr) end -exec_finder(find_ui_building_in_assign, 'ui_building_in_assign') - -- -- ui_building_in_resize -- @@ -531,7 +525,7 @@ Searching for ui_building_in_resize. Please activate the 'q' mode, select a room building (e.g. a bedroom) and do as instructed below. -NOTE: After first 3 steps resize the game window.]], +NOTE: If not done after first 3-4 steps, resize the game window.]], 'int8_t', { 1, 0 }, { [1] = 'enter the Resize room mode', @@ -540,9 +534,6 @@ NOTE: After first 3 steps resize the game window.]], ms.found_offset('ui_building_in_resize', addr) end -exec_finder(find_ui_building_in_resize, 'ui_building_in_resize') - - -- -- window_x -- @@ -557,8 +548,6 @@ scroll to the LEFT edge, then do as instructed:]], ms.found_offset('window_x', addr) end -exec_finder(find_window_x, 'window_x') - -- -- window_y -- @@ -573,8 +562,6 @@ scroll to the TOP edge, then do as instructed:]], ms.found_offset('window_y', addr) end -exec_finder(find_window_y, 'window_y') - -- -- window_z -- @@ -582,20 +569,47 @@ exec_finder(find_window_y, 'window_y') local function find_window_z() local addr = searcher:find_counter([[ Searching for window_z. Please exit to main dwarfmode menu, -scroll to ground level, then do as instructed below. +scroll to a Z level near surface, then do as instructed below. -NOTE: After first 3 steps resize the game window.]], +NOTE: If not done after first 3-4 steps, resize the game window.]], 'int32_t', -1, "Please press '>' to scroll one Z level down." ) ms.found_offset('window_z', addr) end -exec_finder(find_window_z, 'window_z') - -- --- THE END +-- MAIN FLOW -- -print('Done.') +print('\nInitial globals (need title screen):\n') + +exec_finder(find_cursor, { 'cursor', 'selection_rect', 'gamemode', 'gametype' }) +exec_finder(find_announcements, 'announcements') +exec_finder(find_d_init, 'd_init') +exec_finder(find_gview, 'gview') + +print('\nCompound globals (need loaded world):\n') + +exec_finder(find_world, 'world') +exec_finder(find_ui, 'ui') +exec_finder(find_ui_sidebar_menus, 'ui_sidebar_menus') +exec_finder(find_ui_build_selector, 'ui_build_selector') + +print('\nPrimitive globals:\n') + +exec_finder(find_ui_menu_width, { 'ui_menu_width', 'ui_area_map_width' }) +exec_finder(find_ui_selected_unit, 'ui_selected_unit') +exec_finder(find_ui_unit_view_mode, 'ui_unit_view_mode') +exec_finder(find_ui_look_cursor, 'ui_look_cursor') +exec_finder(find_ui_building_item_cursor, 'ui_building_item_cursor') +exec_finder(find_ui_workshop_in_add, 'ui_workshop_in_add') +exec_finder(find_ui_workshop_job_cursor, 'ui_workshop_job_cursor') +exec_finder(find_ui_building_in_assign, 'ui_building_in_assign') +exec_finder(find_ui_building_in_resize, 'ui_building_in_resize') +exec_finder(find_window_x, 'window_x') +exec_finder(find_window_y, 'window_y') +exec_finder(find_window_z, 'window_z') + +print('\nDone. Now add newly-found globals to symbols.xml.') searcher:reset() From 50dff568994a79feaa465b03da9e251ef87c0798 Mon Sep 17 00:00:00 2001 From: Alexander Gavrilov Date: Tue, 19 Jun 2012 18:41:18 +0400 Subject: [PATCH 19/28] Rewrite getMemRanges for windows to get rid of a number of problems. - Properly handle copy-on-write permission modes. - Merge ranges with the same properties for us. - Don't skip non-private areas. - Use the mapped filename as name, so that it works for all ranges. --- library/Process-windows.cpp | 136 ++++++++++++++++++++++++++++++------ 1 file changed, 113 insertions(+), 23 deletions(-) diff --git a/library/Process-windows.cpp b/library/Process-windows.cpp index 944a773c6..7eb6ff5f7 100644 --- a/library/Process-windows.cpp +++ b/library/Process-windows.cpp @@ -233,19 +233,55 @@ struct HeapBlock ULONG reserved; }; */ -// FIXME: NEEDS TESTING! -// FIXME: i noticed that if you enumerate it twice, second time it returns wrong .text region size + +static void GetDosNames(std::map &table) +{ + // Partially based on example from msdn: + // Translate path with device name to drive letters. + TCHAR szTemp[512]; + szTemp[0] = '\0'; + + if (GetLogicalDriveStrings(sizeof(szTemp)-1, szTemp)) + { + TCHAR szName[MAX_PATH]; + TCHAR szDrive[3] = " :"; + BOOL bFound = FALSE; + TCHAR* p = szTemp; + + do + { + // Copy the drive letter to the template string + *szDrive = *p; + + // Look up each device name + if (QueryDosDevice(szDrive, szName, MAX_PATH)) + table[szName] = szDrive; + + // Go to the next NULL character. + while (*p++); + } while (*p); // end of string + } +} + void Process::getMemRanges( vector & ranges ) { MEMORY_BASIC_INFORMATION MBI; //map heaps; uint64_t movingStart = 0; + PVOID LastAllocationBase = 0; map nameMap; + map dosDrives; // get page size SYSTEM_INFO si; GetSystemInfo(&si); uint64_t PageSize = si.dwPageSize; + + // get dos drive names + GetDosNames(dosDrives); + + ranges.clear(); + // enumerate heaps // HeapNodes(d->my_pid, heaps); // go through all the VM regions, convert them to our internal format @@ -254,52 +290,106 @@ void Process::getMemRanges( vector & ranges ) movingStart = ((uint64_t)MBI.BaseAddress + MBI.RegionSize); if(movingStart % PageSize != 0) movingStart = (movingStart / PageSize + 1) * PageSize; - // skip empty regions and regions we share with other processes (DLLs) - if( !(MBI.State & MEM_COMMIT) /*|| !(MBI.Type & MEM_PRIVATE)*/ ) + + // Skip unallocated address space + if (MBI.State & MEM_FREE) continue; + + // Find range and permissions t_memrange temp; + memset(&temp, 0, sizeof(temp)); + temp.start = (char *) MBI.BaseAddress; temp.end = ((char *)MBI.BaseAddress + (uint64_t)MBI.RegionSize); - temp.read = MBI.Protect & PAGE_EXECUTE_READ || MBI.Protect & PAGE_EXECUTE_READWRITE || MBI.Protect & PAGE_READONLY || MBI.Protect & PAGE_READWRITE; - temp.write = MBI.Protect & PAGE_EXECUTE_READWRITE || MBI.Protect & PAGE_READWRITE; - temp.execute = MBI.Protect & PAGE_EXECUTE_READ || MBI.Protect & PAGE_EXECUTE_READWRITE || MBI.Protect & PAGE_EXECUTE; - temp.valid = true; - if(!GetModuleBaseName(d->my_handle, (HMODULE) temp.start, temp.name, 1024)) + temp.valid = true; + + if (!(MBI.State & MEM_COMMIT)) + temp.valid = false; // reserved address space + else if (MBI.Protect & PAGE_EXECUTE) + temp.execute = true; + else if (MBI.Protect & PAGE_EXECUTE_READ) + temp.execute = temp.read = true; + else if (MBI.Protect & PAGE_EXECUTE_READWRITE) + temp.execute = temp.read = temp.write = true; + else if (MBI.Protect & PAGE_EXECUTE_WRITECOPY) + temp.execute = temp.read = temp.write = true; + else if (MBI.Protect & PAGE_READONLY) + temp.read = true; + else if (MBI.Protect & PAGE_READWRITE) + temp.read = temp.write = true; + else if (MBI.Protect & PAGE_WRITECOPY) + temp.read = temp.write = true; + + // Merge areas with the same properties + if (!ranges.empty() && LastAllocationBase == MBI.AllocationBase) { - if(nameMap.count((char *)temp.start)) + auto &last = ranges.back(); + + if (last.end == temp.start && + last.valid == temp.valid && last.execute == temp.execute && + last.read == temp.read && last.write == temp.write) { - // potential buffer overflow... - strcpy(temp.name, nameMap[(char *)temp.start].c_str()); + last.end = temp.end; + continue; } - else + } + +#if 1 + // Find the mapped file name + if (GetMappedFileName(d->my_handle, temp.start, temp.name, 1024)) + { + int vsize = strlen(temp.name); + + // Translate NT name to DOS name + for (auto it = dosDrives.begin(); it != dosDrives.end(); ++it) { - // filter away shared segments without a name. - if( !(MBI.Type & MEM_PRIVATE) ) + int ksize = it->first.size(); + if (strncmp(temp.name, it->first.data(), ksize) != 0) continue; - else - temp.name[0]=0; + + memcpy(temp.name, it->second.data(), it->second.size()); + memmove(temp.name + it->second.size(), temp.name + ksize, vsize + 1 - ksize); + break; } } else + temp.name[0] = 0; +#else + // Find the executable name + char *base = (char*)MBI.AllocationBase; + + if(nameMap.count(base)) + { + strncpy(temp.name, nameMap[base].c_str(), 1023); + } + else if(GetModuleBaseName(d->my_handle, (HMODULE)base, temp.name, 1024)) { + std::string nm(temp.name); + + nameMap[base] = nm; + // this is our executable! (could be generalized to pull segments from libs, but whatever) - if(d->base == temp.start) + if(d->base == base) { for(int i = 0; i < d->pe_header.FileHeader.NumberOfSections; i++) { - char sectionName[9]; + /*char sectionName[9]; memcpy(sectionName,d->sections[i].Name,8); sectionName[8] = 0; string nm; nm.append(temp.name); nm.append(" : "); - nm.append(sectionName); - nameMap[(char *)temp.start + d->sections[i].VirtualAddress] = nm; + nm.append(sectionName);*/ + nameMap[base + d->sections[i].VirtualAddress] = nm; } } - else - continue; } + else + temp.name[0] = 0; +#endif + + // Push the entry + LastAllocationBase = MBI.AllocationBase; ranges.push_back(temp); } } From 50bd758876adb8a39c54f107af9a5bd2274a3638 Mon Sep 17 00:00:00 2001 From: Alexander Gavrilov Date: Tue, 19 Jun 2012 18:48:22 +0400 Subject: [PATCH 20/28] Replace dfhack.internal.getBase with getRebaseDelta. Also, when printing found offsets, subtract the delta. --- LUA_API.rst | 4 ++-- library/LuaApi.cpp | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/LUA_API.rst b/LUA_API.rst index 1723711d3..9515690eb 100644 --- a/LUA_API.rst +++ b/LUA_API.rst @@ -1172,9 +1172,9 @@ and are only documented here for completeness: Returns the pre-extracted vtable address ``name``, or *nil*. -* ``dfhack.internal.getBase()`` +* ``dfhack.internal.getRebaseDelta()`` - Returns the base address of the process. + Returns the ASLR rebase offset of the DF executable. * ``dfhack.internal.getMemRanges()`` diff --git a/library/LuaApi.cpp b/library/LuaApi.cpp index 631b3c499..3693070d0 100644 --- a/library/LuaApi.cpp +++ b/library/LuaApi.cpp @@ -1036,10 +1036,10 @@ static void *checkaddr(lua_State *L, int idx, bool allow_null = false) return rv; } -static uint32_t getBase() { return Core::getInstance().p->getBase(); } +static uint32_t getRebaseDelta() { return Core::getInstance().vinfo->getRebaseDelta(); } static const LuaWrapper::FunctionReg dfhack_internal_module[] = { - WRAP(getBase), + WRAP(getRebaseDelta), { NULL, NULL } }; @@ -1074,6 +1074,7 @@ static int internal_setAddress(lua_State *L) } // Print via printerr, so that it is definitely logged to stderr.log. + addr -= Core::getInstance().vinfo->getRebaseDelta(); std::string msg = stl_sprintf("", name.c_str(), addr); dfhack_printerr(L, msg); From e687a07f2e463472ad4609371f7b914b5de649a6 Mon Sep 17 00:00:00 2001 From: Alexander Gavrilov Date: Tue, 19 Jun 2012 21:02:27 +0400 Subject: [PATCH 21/28] Fix getRebaseDelta: should be signed int. --- Lua API.html | 4 ++-- library/LuaApi.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lua API.html b/Lua API.html index f1bdd17d2..84d13e2f0 100644 --- a/Lua API.html +++ b/Lua API.html @@ -1329,8 +1329,8 @@ global environment, persistent between calls to the script.

  • dfhack.internal.getVTable(name)

    Returns the pre-extracted vtable address name, or nil.

    • -

  • dfhack.internal.getBase()

    -

    Returns the base address of the process.

    +

  • dfhack.internal.getRebaseDelta()

    +

    Returns the ASLR rebase offset of the DF executable.

  • dfhack.internal.getMemRanges()

    Returns a sequence of tables describing virtual memory ranges of the process.

    diff --git a/library/LuaApi.cpp b/library/LuaApi.cpp index 3693070d0..092404e33 100644 --- a/library/LuaApi.cpp +++ b/library/LuaApi.cpp @@ -1036,7 +1036,7 @@ static void *checkaddr(lua_State *L, int idx, bool allow_null = false) return rv; } -static uint32_t getRebaseDelta() { return Core::getInstance().vinfo->getRebaseDelta(); } +static int getRebaseDelta() { return Core::getInstance().vinfo->getRebaseDelta(); } static const LuaWrapper::FunctionReg dfhack_internal_module[] = { WRAP(getRebaseDelta), From f989ef213e69fc2806ac01fa8f98ad88e7e1fcde Mon Sep 17 00:00:00 2001 From: Alexander Gavrilov Date: Tue, 19 Jun 2012 22:43:31 +0400 Subject: [PATCH 22/28] Add 5 more finders to the script; these use relative location heuristics. --- scripts/devel/find-offsets.lua | 159 ++++++++++++++++++++++++++++++++- 1 file changed, 158 insertions(+), 1 deletion(-) diff --git a/scripts/devel/find-offsets.lua b/scripts/devel/find-offsets.lua index fddfbc948..ef9c98299 100644 --- a/scripts/devel/find-offsets.lua +++ b/scripts/devel/find-offsets.lua @@ -5,6 +5,8 @@ local ms = require 'memscan' local is_known = dfhack.internal.getAddress +local os_type = dfhack.getOSType() + local force_scan = {} for _,v in ipairs({...}) do force_scan[v] = true @@ -56,6 +58,34 @@ local function validate_offset(name,validator,addr,tname,...) ms.found_offset(name,obj) end +local function zoomed_searcher(startn, end_or_sz) + if force_scan.nozoom then + return nil + end + local sv = is_known(startn) + if not sv then + return nil + end + local ev + if type(end_or_sz) == 'number' then + ev = sv + end_or_sz + if end_or_sz < 0 then + sv, ev = ev, sv + end + else + ev = is_known(end_or_sz) + if not ev then + return nil + end + end + sv = sv - (sv % 4) + ev = ev + 3 + ev = ev - (ev % 4) + if data:contains_range(sv, ev-sv) then + return ms.DiffSearcher.new(ms.MemoryArea.new(sv,ev)) + end +end + local function exec_finder(finder, names) if type(names) ~= 'table' then names = { names } @@ -80,7 +110,8 @@ end local ordinal_names = { [0] = '1st entry', - [1] = '2nd entry' + [1] = '2nd entry', + [2] = '3rd entry' } setmetatable(ordinal_names, { __index = function(self,idx) return (idx+1)..'th entry' end @@ -578,6 +609,124 @@ NOTE: If not done after first 3-4 steps, resize the game window.]], ms.found_offset('window_z', addr) end +-- +-- cur_year +-- + +local function find_cur_year() + local zone + if os_type == 'windows' then + zone = zoomed_searcher('formation_next_id', 32) + elseif os_type == 'darwin' then + zone = zoomed_searcher('cursor', -32) + elseif os_type == 'linux' then + zone = zoomed_searcher('ui_building_assign_type', -512) + end + if not zone then + dfhack.printerr('Cannot search for cur_year - prerequisites missing.') + return + end + + local yvalue = utils.prompt_input('Please enter current in-game year: ', utils.check_number) + local idx, addr = zone.area.int32_t:find_one{yvalue} + if idx then + ms.found_offset('cur_year', addr) + return + end + + dfhack.printerr('Could not find cur_year') +end + +-- +-- cur_year_tick +-- + +local function find_cur_year_tick() + local zone + if os_type == 'windows' then + zone = zoomed_searcher('artifact_next_id', -32) + else + zone = zoomed_searcher('cur_year', 128) + end + if not zone then + dfhack.printerr('Cannot search for cur_year_tick - prerequisites missing.') + return + end + + local addr = zone:find_counter([[ +Searching for cur_year_tick. Please exit to main dwarfmode +menu, then do as instructed below:]], + 'int32_t', 1, + "Please press '.' to step the game one frame." + ) + ms.found_offset('cur_year_tick', addr) +end + +-- +-- process_jobs +-- + +local function get_process_zone() + if os_type == 'windows' then + return zoomed_searcher('ui_workshop_job_cursor', 'ui_building_in_resize') + else + return zoomed_searcher('cur_year', 'cur_year_tick') + end +end + +local function find_process_jobs() + local zone = get_process_zone() or searcher + + local addr = zone:find_menu_cursor([[ +Searching for process_jobs. Please do as instructed below:]], + 'int8_t', + { 1, 0 }, + { [1] = 'designate a building to be constructed, e.g a bed', + [0] = 'step or unpause the game to reset the flag' } + ) + ms.found_offset('process_jobs', addr) +end + +-- +-- process_dig +-- + +local function find_process_dig() + local zone = get_process_zone() or searcher + + local addr = zone:find_menu_cursor([[ +Searching for process_dig. Please do as instructed below:]], + 'int8_t', + { 1, 0 }, + { [1] = 'designate a tile to be mined out', + [0] = 'step or unpause the game to reset the flag' } + ) + ms.found_offset('process_dig', addr) +end + +-- +-- pause_state +-- + +local function find_pause_state() + local zone + if os_type == 'linux' then + zone = zoomed_searcher('ui_look_cursor', 32) + elseif os_type == 'windows' then + zone = zoomed_searcher('ui_workshop_job_cursor', 64) + end + zone = zone or searcher + + local addr = zone:find_menu_cursor([[ +Searching for pause_state. Please do as instructed below:]], + 'int8_t', + { 1, 0 }, + { [1] = 'PAUSE the game', + [0] = 'UNPAUSE the game' } + ) + ms.found_offset('pause_state', addr) +end + -- -- MAIN FLOW -- @@ -611,5 +760,13 @@ exec_finder(find_window_x, 'window_x') exec_finder(find_window_y, 'window_y') exec_finder(find_window_z, 'window_z') +print('\nUnpausing globals:\n') + +exec_finder(find_cur_year, 'cur_year') +exec_finder(find_cur_year_tick, 'cur_year_tick') +exec_finder(find_process_jobs, 'process_jobs') +exec_finder(find_process_dig, 'process_dig') +exec_finder(find_pause_state, 'pause_state') + print('\nDone. Now add newly-found globals to symbols.xml.') searcher:reset() From ed4acbdedbb0ddecac5b8b666ef159289697ec93 Mon Sep 17 00:00:00 2001 From: Alexander Gavrilov Date: Wed, 20 Jun 2012 10:12:26 +0400 Subject: [PATCH 23/28] Add a searcher for current_weather, using a prepared save. --- library/lua/memscan.lua | 2 +- scripts/devel/find-offsets.lua | 49 +++++++++++++++++++++-- scripts/devel/prepare-save.lua | 71 ++++++++++++++++++++++++++++++++++ 3 files changed, 117 insertions(+), 5 deletions(-) create mode 100644 scripts/devel/prepare-save.lua diff --git a/library/lua/memscan.lua b/library/lua/memscan.lua index 4bd01c8f7..65b02194c 100644 --- a/library/lua/memscan.lua +++ b/library/lua/memscan.lua @@ -168,7 +168,7 @@ function MemoryArea:__tostring() return string.format('', self.start_addr, self.end_addr) end function MemoryArea:contains_range(start,size) - return start >= self.start_addr and (start+size) <= self.end_addr + return size >= 0 and start >= self.start_addr and (start+size) <= self.end_addr end function MemoryArea:contains_obj(obj,count) local size, base = df.sizeof(obj) diff --git a/scripts/devel/find-offsets.lua b/scripts/devel/find-offsets.lua index ef9c98299..391390f14 100644 --- a/scripts/devel/find-offsets.lua +++ b/scripts/devel/find-offsets.lua @@ -24,7 +24,10 @@ PERMANENT SAVE CORRUPTION. Finding the first few globals requires this script to be started immediately after loading the game, WITHOUT -first loading a world. +first loading a world. The rest expect a loaded save, +not a fresh embark. Finding current_weather requires +a special save previously processed with devel/prepare-save +on a DF version with working dfhack. The script expects vanilla game configuration, without any custom tilesets or init file changes. Never unpause @@ -387,6 +390,43 @@ number, so when it shows "Min (5000df", it means 50000:]], addr, df.ui_build_selector, 'plate_info', 'unit_min') end +-- +-- current_weather +-- + +local function find_current_weather() + print('\nPlease load the save previously processed with prepare-save.') + if not utils.prompt_yes_no('Proceed?', true) then + return + end + + local zone + if os_type == 'windows' then + zone = zoomed_searcher('crime_next_id', 512) + elseif os_type == 'darwin' then + zone = zoomed_searcher('cursor', -64) + elseif os_type == 'linux' then + zone = zoomed_searcher('ui_building_assign_type', -512) + end + zone = zone or searcher + + local wbytes = { + 2, 1, 0, 2, 0, + 1, 2, 1, 0, 0, + 2, 0, 2, 1, 2, + 1, 2, 0, 1, 1, + 2, 0, 1, 0, 2 + } + + local idx, addr = zone.area.int8_t:find_one(wbytes) + if idx then + ms.found_offset('current_weather', addr) + return + end + + dfhack.printerr('Could not find current_weather - must be a wrong save.') +end + -- -- ui_menu_width -- @@ -669,7 +709,7 @@ end local function get_process_zone() if os_type == 'windows' then return zoomed_searcher('ui_workshop_job_cursor', 'ui_building_in_resize') - else + elseif os_type == 'linux' or os_type == 'darwin' then return zoomed_searcher('cur_year', 'cur_year_tick') end end @@ -710,10 +750,10 @@ end local function find_pause_state() local zone - if os_type == 'linux' then + if os_type == 'linux' or os_type == 'darwin' then zone = zoomed_searcher('ui_look_cursor', 32) elseif os_type == 'windows' then - zone = zoomed_searcher('ui_workshop_job_cursor', 64) + zone = zoomed_searcher('ui_workshop_job_cursor', 80) end zone = zone or searcher @@ -747,6 +787,7 @@ exec_finder(find_ui_build_selector, 'ui_build_selector') print('\nPrimitive globals:\n') +exec_finder(find_current_weather, 'current_weather') exec_finder(find_ui_menu_width, { 'ui_menu_width', 'ui_area_map_width' }) exec_finder(find_ui_selected_unit, 'ui_selected_unit') exec_finder(find_ui_unit_view_mode, 'ui_unit_view_mode') diff --git a/scripts/devel/prepare-save.lua b/scripts/devel/prepare-save.lua new file mode 100644 index 000000000..781e3b892 --- /dev/null +++ b/scripts/devel/prepare-save.lua @@ -0,0 +1,71 @@ +-- Prepare the current save for use with devel/find-offsets. + +df.global.pause_state = true + +--[[print('Placing anchor...') + +do + local wp = df.global.ui.waypoints + + for _,pt in ipairs(wp.points) do + if pt.name == 'dfhack_anchor' then + print('Already placed.') + goto found + end + end + + local x,y,z = pos2xyz(df.global.cursor) + + if not x then + error("Place cursor at your preferred anchor point.") + end + + local id = wp.next_point_id + wp.next_point_id = id + 1 + + wp.points:insert('#',{ + new = true, id = id, name = 'dfhack_anchor', + comment=(x..','..y..','..z), + tile = string.byte('!'), fg_color = COLOR_LIGHTRED, bg_color = COLOR_BLUE, + pos = xyz2pos(x,y,z) + }) + +::found:: +end]] + +print('Nicknaming units...') + +for i,unit in ipairs(df.global.world.units.active) do + dfhack.units.setNickname(unit, i..':'..unit.id) +end + +print('Setting weather...') + +local wbytes = { + 2, 1, 0, 2, 0, + 1, 2, 1, 0, 0, + 2, 0, 2, 1, 2, + 1, 2, 0, 1, 1, + 2, 0, 1, 0, 2 +} + +for i=0,4 do + for j = 0,4 do + df.global.current_weather[i][j] = (wbytes[i*5+j+1] or 2) + end +end + +local yearstr = df.global.cur_year..','..df.global.cur_year_tick + +print('Cur year and tick: '..yearstr) + +dfhack.persistent.save{ + key='prepare-save/cur_year', + value=yearstr, + ints={df.global.cur_year, df.global.cur_year_tick} +} + +-- Save + +dfhack.run_script('quicksave') + From f207714d4225e729fa43d04048d595ad6954521d Mon Sep 17 00:00:00 2001 From: Alexander Gavrilov Date: Thu, 21 Jun 2012 21:08:36 +0400 Subject: [PATCH 24/28] Add finders for enabler, gps and init. --- library/lua/memscan.lua | 29 +++++++- scripts/devel/find-offsets.lua | 125 +++++++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+), 1 deletion(-) diff --git a/library/lua/memscan.lua b/library/lua/memscan.lua index 65b02194c..4cf8d41c8 100644 --- a/library/lua/memscan.lua +++ b/library/lua/memscan.lua @@ -154,7 +154,8 @@ function MemoryArea.new(astart, aend) int16_t = CheckedArray.new('int16_t',astart,aend), uint16_t = CheckedArray.new('uint16_t',astart,aend), int32_t = CheckedArray.new('int32_t',astart,aend), - uint32_t = CheckedArray.new('uint32_t',astart,aend) + uint32_t = CheckedArray.new('uint32_t',astart,aend), + float = CheckedArray.new('float',astart,aend) } setmetatable(obj, MemoryArea) return obj @@ -453,4 +454,30 @@ function DiffSearcher:find_counter(prompt,data_type,delta,action_prompt) ) end +-- Screen size + +function get_screen_size() + -- Use already known globals + if dfhack.internal.getAddress('init') then + local d = df.global.init.display + return d.grid_x, d.grid_y + end + if dfhack.internal.getAddress('gps') then + local g = df.global.gps + return g.dimx, g.dimy + end + + -- Parse stdout.log for resize notifications + io.stdout:flush() + + local w,h = 80,25 + for line in io.lines('stdout.log') do + local cw, ch = string.match(line, '^Resizing grid to (%d+)x(%d+)$') + if cw and ch then + w, h = tonumber(cw), tonumber(ch) + end + end + return w,h +end + return _ENV diff --git a/scripts/devel/find-offsets.lua b/scripts/devel/find-offsets.lua index 391390f14..bddd29dfe 100644 --- a/scripts/devel/find-offsets.lua +++ b/scripts/devel/find-offsets.lua @@ -252,6 +252,83 @@ local function find_gview() dfhack.printerr('Could not find gview') end +-- +-- enabler +-- + +local function is_valid_enabler(e) + if not ms.is_valid_vector(e.textures.raws, 4) + or not ms.is_valid_vector(e.text_system, 4) + then + dfhack.printerr('Vector layout check failed.') + return false + end + + return true +end + +local function find_enabler() + -- Data from data/init/colors.txt + local colors = { + 0, 0, 0, 0, 0, 128, 0, 128, 0, + 0, 128, 128, 128, 0, 0, 128, 0, 128, + 128, 128, 0, 192, 192, 192, 128, 128, 128, + 0, 0, 255, 0, 255, 0, 0, 255, 255, + 255, 0, 0, 255, 0, 255, 255, 255, 0, + 255, 255, 255 + } + + for i = 1,#colors do colors[i] = colors[i]/255 end + + local idx, addr = data.float:find_one(colors) + if idx then + validate_offset('enabler', is_valid_enabler, addr, df.enabler, 'ccolor') + return + end + + dfhack.printerr('Could not find enabler') +end + +-- +-- gps +-- + +local function is_valid_gps(g) + if g.clipx[0] < 0 or g.clipx[0] > g.clipx[1] or g.clipx[1] >= g.dimx then + dfhack.printerr('Invalid clipx: ', g.clipx[0], g.clipx[1], g.dimx) + end + if g.clipy[0] < 0 or g.clipy[0] > g.clipy[1] or g.clipy[1] >= g.dimy then + dfhack.printerr('Invalid clipy: ', g.clipy[0], g.clipy[1], g.dimy) + end + + return true +end + +local function find_gps() + print('\nPlease ensure the mouse cursor is not over the game window.') + if not utils.prompt_yes_no('Proceed?', true) then + return + end + + local zone + if os_type == 'windows' or os_type == 'linux' then + zone = zoomed_searcher('cursor', 0x1000) + elseif os_type == 'darwin' then + zone = zoomed_searcher('enabler', 0x1000) + end + zone = zone or searcher + + local w,h = ms.get_screen_size() + + local idx, addr = zone.area.int32_t:find_one{w, h, -1, -1} + if idx then + validate_offset('gps', is_valid_gps, addr, df.graphic, 'dimx') + return + end + + dfhack.printerr('Could not find gps') +end + -- -- World -- @@ -390,6 +467,51 @@ number, so when it shows "Min (5000df", it means 50000:]], addr, df.ui_build_selector, 'plate_info', 'unit_min') end +-- +-- init +-- + +local function is_valid_init(i) + -- derived from curses_*.png image sizes presumably + if i.font.small_font_dispx ~= 8 or i.font.small_font_dispy ~= 12 or + i.font.large_font_dispx ~= 10 or i.font.large_font_dispy ~= 12 then + print('Unexpected font sizes: ', + i.font.small_font_dispx, i.font.small_font_dispy, + i.font.large_font_dispx, i.font.large_font_dispy) + if not utils.prompt_yes_no('Ignore?') then + return false + end + end + + return true +end + +local function find_init() + local zone + if os_type == 'windows' then + zone = zoomed_searcher('ui_build_selector', 0x3000) + elseif os_type == 'linux' or os_type == 'darwin' then + zone = zoomed_searcher('d_init', -0x2000) + end + zone = zone or searcher + + local idx, addr = zone.area.int32_t:find_one{250, 150, 15, 0} + if idx then + validate_offset('init', is_valid_init, addr, df.init, 'input', 'hold_time') + return + end + + local w,h = ms.get_screen_size() + + local idx, addr = zone.area.int32_t:find_one{w, h} + if idx then + validate_offset('init', is_valid_init, addr, df.init, 'display', 'grid_x') + return + end + + dfhack.printerr('Could not find init') +end + -- -- current_weather -- @@ -777,6 +899,8 @@ exec_finder(find_cursor, { 'cursor', 'selection_rect', 'gamemode', 'gametype' }) exec_finder(find_announcements, 'announcements') exec_finder(find_d_init, 'd_init') exec_finder(find_gview, 'gview') +exec_finder(find_enabler, 'enabler') +exec_finder(find_gps, 'gps') print('\nCompound globals (need loaded world):\n') @@ -784,6 +908,7 @@ exec_finder(find_world, 'world') exec_finder(find_ui, 'ui') exec_finder(find_ui_sidebar_menus, 'ui_sidebar_menus') exec_finder(find_ui_build_selector, 'ui_build_selector') +exec_finder(find_init, 'init') print('\nPrimitive globals:\n') From 752da9ced5ce2df8cc9638cbf75a769325540e31 Mon Sep 17 00:00:00 2001 From: Alexander Gavrilov Date: Thu, 21 Jun 2012 21:26:25 +0400 Subject: [PATCH 25/28] Move formatting newly-found globals for symbols.xml to lua code. --- LUA_API.rst | 1 + Lua API.html | 3 ++- library/LuaApi.cpp | 6 +++--- library/LuaTools.cpp | 5 ++++- library/include/ColorText.h | 2 ++ library/lua/memscan.lua | 10 ++++++++++ 6 files changed, 22 insertions(+), 5 deletions(-) diff --git a/LUA_API.rst b/LUA_API.rst index 9515690eb..5136bba57 100644 --- a/LUA_API.rst +++ b/LUA_API.rst @@ -451,6 +451,7 @@ Currently it defines the following features: * ``dfhack.color([color])`` Sets the current output color. If color is *nil* or *-1*, resets to default. + Returns the previous color value. * ``dfhack.is_interactive()`` diff --git a/Lua API.html b/Lua API.html index 84d13e2f0..1c4dc4059 100644 --- a/Lua API.html +++ b/Lua API.html @@ -734,7 +734,8 @@ works with DFHack output infrastructure.

    Same as println; intended for errors. Uses red color and logs to stderr.log.

  • dfhack.color([color])

    -

    Sets the current output color. If color is nil or -1, resets to default.

    +

    Sets the current output color. If color is nil or -1, resets to default. +Returns the previous color value.

  • dfhack.is_interactive()

    Checks if the thread can access the interactive console and returns true or false.

    diff --git a/library/LuaApi.cpp b/library/LuaApi.cpp index 092404e33..b0a085eca 100644 --- a/library/LuaApi.cpp +++ b/library/LuaApi.cpp @@ -1074,9 +1074,9 @@ static int internal_setAddress(lua_State *L) } // Print via printerr, so that it is definitely logged to stderr.log. - addr -= Core::getInstance().vinfo->getRebaseDelta(); - std::string msg = stl_sprintf("", name.c_str(), addr); - dfhack_printerr(L, msg); + uint32_t iaddr = addr - Core::getInstance().vinfo->getRebaseDelta(); + fprintf(stderr, "Setting global '%s' to %x (%x)\n", name.c_str(), addr, iaddr); + fflush(stderr); return 1; } diff --git a/library/LuaTools.cpp b/library/LuaTools.cpp index 752c341b2..48244dedf 100644 --- a/library/LuaTools.cpp +++ b/library/LuaTools.cpp @@ -256,8 +256,11 @@ static int lua_dfhack_color(lua_State *S) luaL_argerror(S, 1, "invalid color value"); color_ostream *out = Lua::GetOutput(S); - if (out) + if (out) { + lua_pushinteger(S, (int)out->color()); out->color(color_ostream::color_value(cv)); + return 1; + } return 0; } diff --git a/library/include/ColorText.h b/library/include/ColorText.h index 105832efd..0cc286dcf 100644 --- a/library/include/ColorText.h +++ b/library/include/ColorText.h @@ -111,6 +111,8 @@ namespace DFHack void printerr(const char *format, ...); void vprinterr(const char *format, va_list args); + /// Get color + color_value color() { return cur_color; } /// Set color (ANSI color number) void color(color_value c); /// Reset color to default diff --git a/library/lua/memscan.lua b/library/lua/memscan.lua index 4cf8d41c8..92a3e3e80 100644 --- a/library/lua/memscan.lua +++ b/library/lua/memscan.lua @@ -252,6 +252,16 @@ function found_offset(name,val) end else dfhack.internal.setAddress(name, val) + + local ival = val - dfhack.internal.getRebaseDelta() + local entry = string.format("\n", name, ival) + + local ccolor = dfhack.color(COLOR_LIGHTGREEN) + dfhack.print(entry) + dfhack.color(ccolor) + + io.stdout:write(entry) + io.stdout:flush() end end From 65e82f7c12f95e461363e15c781c3fd4c5d241d3 Mon Sep 17 00:00:00 2001 From: Alexander Gavrilov Date: Fri, 22 Jun 2012 16:36:50 +0400 Subject: [PATCH 26/28] Support controllable error presentation verbosity in lua code. Use qerror to squash stack traces and location prefix. --- LUA_API.rst | 88 +++++++++++++----- Lua API.html | 162 ++++++++++++++++++++++----------- library/LuaTools.cpp | 78 ++++++++++++++-- library/lua/dfhack.lua | 4 + library/lua/memscan.lua | 2 +- library/lua/utils.lua | 2 +- scripts/devel/find-offsets.lua | 6 +- scripts/fix/item-occupancy.lua | 3 +- scripts/quicksave.lua | 3 +- 9 files changed, 259 insertions(+), 89 deletions(-) diff --git a/LUA_API.rst b/LUA_API.rst index 5136bba57..e8c413fe7 100644 --- a/LUA_API.rst +++ b/LUA_API.rst @@ -426,13 +426,17 @@ not destroy any objects allocated in this way, so the user should be prepared to catch the error and do the necessary cleanup. -================ -DFHack utilities -================ +========== +DFHack API +========== DFHack utility functions are placed in the ``dfhack`` global tree. -Currently it defines the following features: +Native utilities +================ + +Input & Output +-------------- * ``dfhack.print(args...)`` @@ -474,23 +478,9 @@ Currently it defines the following features: If the interactive console is not accessible, returns *nil, error*. -* ``dfhack.pcall(f[,args...])`` - - Invokes f via xpcall, using an error function that attaches - a stack trace to the error. The same function is used by SafeCall - in C++, and dfhack.safecall. - - The returned error is a table with separate ``message`` and - ``stacktrace`` string fields; it implements ``__tostring``. - -* ``safecall(f[,args...])``, ``dfhack.safecall(f[,args...])`` - - Just like pcall, but also prints the error using printerr before - returning. Intended as a convenience function. - -* ``dfhack.saferesume(coroutine[,args...])`` - Compares to coroutine.resume like dfhack.safecall vs pcall. +Miscellaneous +------------- * ``dfhack.run_script(name[,args...])`` @@ -511,6 +501,36 @@ Currently it defines the following features: to group operations together in one big critical section. A plugin can choose to run all lua code inside a C++-side suspend lock. + +Exception handling +------------------ + +* ``dfhack.error(msg[,level[,verbose]])`` + + Throws a dfhack exception object with location and stack trace. + The verbose parameter controls whether the trace is printed by default. + +* ``qerror(msg[,level])`` + + Calls ``dfhack.error()`` with ``verbose`` being *false*. Intended to + be used for user-caused errors in scripts, where stack traces are not + desirable. + +* ``dfhack.pcall(f[,args...])`` + + Invokes f via xpcall, using an error function that attaches + a stack trace to the error. The same function is used by SafeCall + in C++, and dfhack.safecall. + +* ``safecall(f[,args...])``, ``dfhack.safecall(f[,args...])`` + + Just like pcall, but also prints the error using printerr before + returning. Intended as a convenience function. + +* ``dfhack.saferesume(coroutine[,args...])`` + + Compares to coroutine.resume like dfhack.safecall vs pcall. + * ``dfhack.call_with_finalizer(num_cleanup_args,always,cleanup_fn[,cleanup_args...],fn[,args...])`` Invokes ``fn`` with ``args``, and after it returns or throws an @@ -535,9 +555,33 @@ Currently it defines the following features: Calls ``fn(obj,args...)``, then finalizes with ``obj:delete()``. +* ``dfhack.exception`` + + Metatable of error objects used by dfhack. The objects have the + following properties: + + ``err.where`` + The location prefix string, or *nil*. + ``err.message`` + The base message string. + ``err.stacktrace`` + The stack trace string, or *nil*. + ``err.cause`` + A different exception object, or *nil*. + ``err.thread`` + The coroutine that has thrown the exception. + ``err.verbose`` + Boolean, or *nil*; specifies if where and stacktrace should be printed. + ``tostring(err)``, or ``err:tostring([verbose])`` + Converts the exception to string. + +* ``dfhack.exception.verbose`` + + The default value of the ``verbose`` argument of ``err:tostring()``. + Persistent configuration storage -================================ +-------------------------------- This api is intended for storing configuration options in the world itself. It probably should be restricted to data that is world-dependent. @@ -579,7 +623,7 @@ functions can just copy values in memory without doing any actual I/O. However, currently every entry has a 180+-byte dead-weight overhead. Material info lookup -==================== +-------------------- A material info record has fields: diff --git a/Lua API.html b/Lua API.html index 1c4dc4059..47cf08ab6 100644 --- a/Lua API.html +++ b/Lua API.html @@ -333,30 +333,36 @@ ul.auto-toc {
  • Recursive table assignment
    • -
  • DFHack utilities
      -
    • Persistent configuration storage
      • -
    • Material info lookup
      • -
    • C++ function wrappers
        -
      • Gui module
        • -
      • Job module
        • -
      • Units module
        • -
      • Items module
        • -
      • Maps module
        • -
      • Burrows module
        • -
      • Buildings module
        • -
      • Constructions module
        • -
      • Internal API
        • +
      • DFHack API
          +
        • Native utilities
          • -
        • Core interpreter context @@ -717,10 +723,13 @@ should be prepared to catch the error and do the necessary cleanup.

          -
          -

          DFHack utilities

          +
          +

          DFHack API

          DFHack utility functions are placed in the dfhack global tree.

          -

          Currently it defines the following features:

          +
          +

          Native utilities

          +
          +

          Input & Output

          • dfhack.print(args...)

            Output tab-separated args as standard lua print would do, @@ -753,20 +762,11 @@ this, forcing the function to block on input with lock held.

            string, global environment and command-line history file.

            If the interactive console is not accessible, returns nil, error.

            • -

          • dfhack.pcall(f[,args...])

            -

            Invokes f via xpcall, using an error function that attaches -a stack trace to the error. The same function is used by SafeCall -in C++, and dfhack.safecall.

            -

            The returned error is a table with separate message and -stacktrace string fields; it implements __tostring.

            -
            • -

          • safecall(f[,args...]), dfhack.safecall(f[,args...])

            -

            Just like pcall, but also prints the error using printerr before -returning. Intended as a convenience function.

            -
            • -

          • dfhack.saferesume(coroutine[,args...])

            -

            Compares to coroutine.resume like dfhack.safecall vs pcall.

            -
            • +
          +
          +
          +

          Miscellaneous

          +

          • dfhack.run_script(name[,args...])

            Run a lua script in hack/scripts/, as if it was started from dfhack command-line. The name argument should be the name stem, as would be used on the command line. @@ -782,6 +782,32 @@ the lock. It is safe to nest suspends.

            to group operations together in one big critical section. A plugin can choose to run all lua code inside a C++-side suspend lock.

            • +
          +
          +
          +

          Exception handling

          +
            +

          • dfhack.error(msg[,level[,verbose]])

            +

            Throws a dfhack exception object with location and stack trace. +The verbose parameter controls whether the trace is printed by default.

            +
            • +

          • qerror(msg[,level])

            +

            Calls dfhack.error() with verbose being false. Intended to +be used for user-caused errors in scripts, where stack traces are not +desirable.

            +
            • +

          • dfhack.pcall(f[,args...])

            +

            Invokes f via xpcall, using an error function that attaches +a stack trace to the error. The same function is used by SafeCall +in C++, and dfhack.safecall.

            +
            • +

          • safecall(f[,args...]), dfhack.safecall(f[,args...])

            +

            Just like pcall, but also prints the error using printerr before +returning. Intended as a convenience function.

            +
            • +

          • dfhack.saferesume(coroutine[,args...])

            +

            Compares to coroutine.resume like dfhack.safecall vs pcall.

            +

          • dfhack.call_with_finalizer(num_cleanup_args,always,cleanup_fn[,cleanup_args...],fn[,args...])

            Invokes fn with args, and after it returns or throws an error calls cleanup_fn with cleanup_args. Any return values from @@ -801,9 +827,40 @@ Implemented using call_with_final

          • dfhack.with_temp_object(obj,fn[,args...])

            Calls fn(obj,args...), then finalizes with obj:delete().

            • +

          • dfhack.exception

            +

            Metatable of error objects used by dfhack. The objects have the +following properties:

            +
            +
            err.where
            +

            The location prefix string, or nil.

            +
            +
            err.message
            +

            The base message string.

            +
            +
            err.stacktrace
            +

            The stack trace string, or nil.

            +
            +
            err.cause
            +

            A different exception object, or nil.

            +
            +
            err.thread
            +

            The coroutine that has thrown the exception.

            +
            +
            err.verbose
            +

            Boolean, or nil; specifies if where and stacktrace should be printed.

            +
            +
            tostring(err), or err:tostring([verbose])
            +

            Converts the exception to string.

            +
            +
            +
            • +

          • dfhack.exception.verbose

            +

            The default value of the verbose argument of err:tostring().

            +
          +
          -

          Persistent configuration storage

          +

          Persistent configuration storage

          This api is intended for storing configuration options in the world itself. It probably should be restricted to data that is world-dependent.

          Entries are identified by a string key, but it is also possible to manage @@ -838,7 +895,7 @@ functions can just copy values in memory without doing any actual I/O. However, currently every entry has a 180+-byte dead-weight overhead.

          -

          Material info lookup

          +

          Material info lookup

          A material info record has fields:

          • type, index, material

            @@ -881,8 +938,9 @@ Accept dfhack_material_category auto-assign table.

          +
          -

          C++ function wrappers

          +

          C++ function wrappers

          Thin wrappers around C++ functions, similar to the ones for virtual methods. One notable difference is that these explicit wrappers allow argument count adjustment according to the usual lua rules, so trailing false/nil arguments @@ -911,7 +969,7 @@ can be omitted.

        -

        Gui module

        +

        Gui module

        • dfhack.gui.getCurViewscreen()

          Returns the viewscreen that is current in the core.

          @@ -947,7 +1005,7 @@ The is_bright boolean actually seems to invert the brightness.

        -

        Job module

        +

        Job module

        • dfhack.job.cloneJobStruct(job)

          Creates a deep copy of the given job.

          @@ -984,7 +1042,7 @@ a lua list containing them.

        -

        Units module

        +

        Units module

        • dfhack.units.getPosition(unit)

          Returns true x,y,z of the unit, or nil if invalid; may be not equal to unit.pos if caged.

          @@ -1038,7 +1096,7 @@ or raws. The ignore_noble boolean disables the
        -

        Items module

        +

        Items module

        • dfhack.items.getPosition(item)

          Returns true x,y,z of the item, or nil if invalid; may be not equal to item.pos if in inventory.

          @@ -1081,7 +1139,7 @@ Returns false in case of error.

        -

        Maps module

        +

        Maps module

        • dfhack.maps.getSize()

          Returns map size in blocks: x, y, z

          @@ -1122,7 +1180,7 @@ burrows, or the presence of invaders.

        -

        Burrows module

        +

        Burrows module

        • dfhack.burrows.findByName(name)

          Returns the burrow pointer or nil.

          @@ -1157,7 +1215,7 @@ burrows, or the presence of invaders.

        -

        Buildings module

        +

        Buildings module

        • dfhack.buildings.getSize(building)

          Returns width, height, centerx, centery.

          @@ -1297,7 +1355,7 @@ can be determined this way, constructBuilding
        -

        Constructions module

        +

        Constructions module

        • dfhack.constructions.designateNew(pos,type,item_type,mat_index)

          Designates a new construction at given position. If there already is @@ -1313,7 +1371,7 @@ Returns true, was_only_planned if removed; or false if none fo

        -

        Internal API

        +

        Internal API

        These functions are intended for the use by dfhack developers, and are only documented here for completeness:

          @@ -1356,7 +1414,7 @@ Returns: found_index, or nil if end reached.

        -

        Core interpreter context

        +

        Core interpreter context

        While plugins can create any number of interpreter instances, there is one special context managed by dfhack core. It is the only context that can receive events from DF and plugins.

        @@ -1387,7 +1445,7 @@ Using timeout_active(id,nil) cancels the timer
      -

      Event type

      +

      Event type

      An event is just a lua table with a predefined metatable that contains a __call metamethod. When it is invoked, it loops through the table with next and calls all contained values. @@ -1413,14 +1471,14 @@ order using dfhack.safecall.

      -

      Plugins

      +

      Plugins

      DFHack plugins may export native functions and events to lua contexts. They are automatically imported by mkmodule('plugins.<name>'); this means that a lua module file is still necessary for require to read.

      The following plugins have lua support.

      -

      burrows

      +

      burrows

      Implements extended burrow manipulations.

      Events:

        @@ -1458,7 +1516,7 @@ set is the same as used by the command line.

        The lua module file also re-exports functions from dfhack.burrows.

      -

      sort

      +

      sort

      Does not export any native functions as of now. Instead, it calls lua code to perform the actual ordering of list items.

      diff --git a/library/LuaTools.cpp b/library/LuaTools.cpp index 48244dedf..28571a0f7 100644 --- a/library/LuaTools.cpp +++ b/library/LuaTools.cpp @@ -426,10 +426,12 @@ static bool convert_to_exception(lua_State *L, int slevel, lua_State *thread = N // Create a new exception for this thread lua_newtable(L); - luaL_where(L, 1); + luaL_where(L, slevel); + lua_setfield(L, -2, "where"); lua_pushstring(L, "coroutine resume failed"); - lua_concat(L, 2); lua_setfield(L, -2, "message"); + lua_getfield(L, -2, "verbose"); + lua_setfield(L, -2, "verbose"); lua_swap(L); lua_setfield(L, -2, "cause"); } @@ -483,12 +485,57 @@ static int dfhack_onerror(lua_State *L) return 1; } +static int dfhack_error(lua_State *L) +{ + luaL_checkany(L, 1); + lua_settop(L, 3); + int level = std::max(1, luaL_optint(L, 2, 1)); + + lua_pushvalue(L, 1); + + if (convert_to_exception(L, level)) + { + luaL_where(L, level); + lua_setfield(L, -2, "where"); + + if (!lua_isnil(L, 3)) + { + lua_pushvalue(L, 3); + lua_setfield(L, -2, "verbose"); + } + } + + return lua_error(L); +} + static int dfhack_exception_tostring(lua_State *L) { luaL_checktype(L, 1, LUA_TTABLE); + lua_settop(L, 2); + + if (lua_isnil(L, 2)) + { + lua_rawgetp(L, LUA_REGISTRYINDEX, &DFHACK_EXCEPTION_META_TOKEN); + lua_getfield(L, -1, "verbose"); + lua_insert(L, 2); + lua_settop(L, 2); + } + + lua_getfield(L, 1, "verbose"); + + bool verbose = + lua_toboolean(L, 2) || lua_toboolean(L, 3) || + (lua_isnil(L, 2) && lua_isnil(L, 3)); int base = lua_gettop(L); + if (verbose || lua_isnil(L, 3)) + { + lua_getfield(L, 1, "where"); + if (!lua_isstring(L, -1)) + lua_pop(L, 1); + } + lua_getfield(L, 1, "message"); if (!lua_isstring(L, -1)) { @@ -496,15 +543,26 @@ static int dfhack_exception_tostring(lua_State *L) lua_pushstring(L, "(error message is not a string)"); } - lua_pushstring(L, "\n"); - lua_getfield(L, 1, "stacktrace"); - if (!lua_isstring(L, -1)) - lua_pop(L, 2); + if (verbose) + { + lua_pushstring(L, "\n"); + lua_getfield(L, 1, "stacktrace"); + if (!lua_isstring(L, -1)) + lua_pop(L, 2); + } lua_pushstring(L, "\ncaused by:\n"); lua_getfield(L, 1, "cause"); if (lua_isnil(L, -1)) lua_pop(L, 2); + else if (lua_istable(L, -1)) + { + lua_pushcfunction(L, dfhack_exception_tostring); + lua_swap(L); + lua_pushvalue(L, 2); + if (lua_pcall(L, 2, 1, 0) != LUA_OK) + error_tostring(L); + } else error_tostring(L); @@ -655,7 +713,12 @@ static int dfhack_coauxwrap (lua_State *L) { if (Lua::IsSuccess(r)) return lua_gettop(L); else + { + if (lua_checkstack(L, LUA_MINSTACK)) + convert_to_exception(L, 1); + return lua_error(L); + } } static int dfhack_cowrap (lua_State *L) { @@ -1162,6 +1225,7 @@ static const luaL_Reg dfhack_funcs[] = { { "safecall", dfhack_safecall }, { "saferesume", dfhack_saferesume }, { "onerror", dfhack_onerror }, + { "error", dfhack_error }, { "call_with_finalizer", dfhack_call_with_finalizer }, { "with_suspend", lua_dfhack_with_suspend }, { "open_plugin", dfhack_open_plugin }, @@ -1362,6 +1426,8 @@ lua_State *DFHack::Lua::Open(color_ostream &out, lua_State *state) lua_newtable(state); lua_pushcfunction(state, dfhack_exception_tostring); lua_setfield(state, -2, "__tostring"); + lua_pushcfunction(state, dfhack_exception_tostring); + lua_setfield(state, -2, "tostring"); lua_dup(state); lua_rawsetp(state, LUA_REGISTRYINDEX, &DFHACK_EXCEPTION_META_TOKEN); lua_setfield(state, -2, "exception"); diff --git a/library/lua/dfhack.lua b/library/lua/dfhack.lua index 4cdb4c950..d200a6c5c 100644 --- a/library/lua/dfhack.lua +++ b/library/lua/dfhack.lua @@ -49,6 +49,10 @@ function dfhack.pcall(f, ...) return xpcall(f, dfhack.onerror, ...) end +function qerror(msg, level) + dfhack.error(msg, (level or 1) + 1, false) +end + function dfhack.with_finalize(...) return dfhack.call_with_finalizer(0,true,...) end diff --git a/library/lua/memscan.lua b/library/lua/memscan.lua index 92a3e3e80..970f821c2 100644 --- a/library/lua/memscan.lua +++ b/library/lua/memscan.lua @@ -235,7 +235,7 @@ function found_offset(name,val) if not val then print('Could not find offset '..name) if not cval and not utils.prompt_yes_no('Continue with the script?') then - error('User quit') + qerror('User quit') end return end diff --git a/library/lua/utils.lua b/library/lua/utils.lua index 93ee840c4..f303091d6 100644 --- a/library/lua/utils.lua +++ b/library/lua/utils.lua @@ -379,7 +379,7 @@ function prompt_yes_no(msg,default) elseif string.match(rv,'^[Nn]') then return false elseif rv == 'abort' then - error('User abort in utils.prompt_yes_no()') + qerror('User abort in utils.prompt_yes_no()') elseif rv == '' and default ~= nil then return default end diff --git a/scripts/devel/find-offsets.lua b/scripts/devel/find-offsets.lua index bddd29dfe..6fc127351 100644 --- a/scripts/devel/find-offsets.lua +++ b/scripts/devel/find-offsets.lua @@ -43,12 +43,12 @@ end local data = ms.get_data_segment() if not data then - error('Could not find data segment') + qerror('Could not find data segment') end print('\nData section: '..tostring(data)) if data.size < 5000000 then - error('Data segment too short.') + qerror('Data segment too short.') end local searcher = ms.DiffSearcher.new(data) @@ -103,7 +103,7 @@ local function exec_finder(finder, names) if not dfhack.safecall(finder) then if not utils.prompt_yes_no('Proceed with the rest of the script?') then searcher:reset() - error('Quit') + qerror('Quit') end end else diff --git a/scripts/fix/item-occupancy.lua b/scripts/fix/item-occupancy.lua index b5466b7a8..09c6b3030 100644 --- a/scripts/fix/item-occupancy.lua +++ b/scripts/fix/item-occupancy.lua @@ -116,8 +116,7 @@ if opt then if opt == '--fix' then fix = true else - dfhack.printerr('Invalid option: '..opt) - return + qerror('Invalid option: '..opt) end end diff --git a/scripts/quicksave.lua b/scripts/quicksave.lua index c54cc730b..f4886b35b 100644 --- a/scripts/quicksave.lua +++ b/scripts/quicksave.lua @@ -1,8 +1,7 @@ -- Makes the game immediately save the state. if not dfhack.isMapLoaded() then - dfhack.printerr("World and map aren't loaded.") - return + qerror("World and map aren't loaded.") end local ui_main = df.global.ui.main From bd37cc09c525d4db9400e224e4ca3cadcf0eed4c Mon Sep 17 00:00:00 2001 From: Alexander Gavrilov Date: Fri, 22 Jun 2012 20:17:55 +0400 Subject: [PATCH 27/28] Update the Lua API document with info about scripts. --- LUA_API.rst | 183 ++++++++++++++++++++++++++++++----------- Lua API.html | 172 ++++++++++++++++++++++++++------------ library/lua/dfhack.lua | 2 + 3 files changed, 256 insertions(+), 101 deletions(-) diff --git a/LUA_API.rst b/LUA_API.rst index e8c413fe7..5fc653bb3 100644 --- a/LUA_API.rst +++ b/LUA_API.rst @@ -4,9 +4,26 @@ DFHack Lua API .. contents:: -==================== -DF structure wrapper -==================== +The current version of DFHack has extensive support for +the Lua scripting language, providing access to: + +1. Raw data structures used by the game. +2. Many C++ functions for high-level access to these + structures, and interaction with dfhack itself. +3. Some functions exported by C++ plugins. + +Lua code can be used both for writing scripts, which +are treated by DFHack command line prompt almost as +native C++ commands, and invoked by plugins written in c++. + +This document describes native API available to Lua in detail. +For the most part it does not describe utility functions +implemented by Lua files located in hack/lua/... + + +========================= +DF data structure wrapper +========================= DF structures described by the xml files in library/xml are exported to lua code as a tree of objects and functions under the ``df`` global, @@ -479,29 +496,6 @@ Input & Output If the interactive console is not accessible, returns *nil, error*. -Miscellaneous -------------- - -* ``dfhack.run_script(name[,args...])`` - - Run a lua script in hack/scripts/, as if it was started from dfhack command-line. - The ``name`` argument should be the name stem, as would be used on the command line. - Note that the script is re-read from the file every time it is called, and errors - are propagated to the caller. - -* ``dfhack.with_suspend(f[,args...])`` - - Calls ``f`` with arguments after grabbing the DF core suspend lock. - Suspending is necessary for accessing a consistent state of DF memory. - - Returned values and errors are propagated through after releasing - the lock. It is safe to nest suspends. - - Every thread is allowed only one suspend per DF frame, so it is best - to group operations together in one big critical section. A plugin - can choose to run all lua code inside a C++-side suspend lock. - - Exception handling ------------------ @@ -531,30 +525,6 @@ Exception handling Compares to coroutine.resume like dfhack.safecall vs pcall. -* ``dfhack.call_with_finalizer(num_cleanup_args,always,cleanup_fn[,cleanup_args...],fn[,args...])`` - - Invokes ``fn`` with ``args``, and after it returns or throws an - error calls ``cleanup_fn`` with ``cleanup_args``. Any return values from - ``fn`` are propagated, and errors are re-thrown. - - The ``num_cleanup_args`` integer specifies the number of ``cleanup_args``, - and the ``always`` boolean specifies if cleanup should be called in any case, - or only in case of an error. - -* ``dfhack.with_finalize(cleanup_fn,fn[,args...])`` - - Calls ``fn`` with arguments, then finalizes with ``cleanup_fn``. - Implemented using ``call_with_finalizer(0,true,...)``. - -* ``dfhack.with_onerror(cleanup_fn,fn[,args...])`` - - Calls ``fn`` with arguments, then finalizes with ``cleanup_fn`` on any thrown error. - Implemented using ``call_with_finalizer(0,false,...)``. - -* ``dfhack.with_temp_object(obj,fn[,args...])`` - - Calls ``fn(obj,args...)``, then finalizes with ``obj:delete()``. - * ``dfhack.exception`` Metatable of error objects used by dfhack. The objects have the @@ -580,6 +550,46 @@ Exception handling The default value of the ``verbose`` argument of ``err:tostring()``. +Locking and finalization +------------------------ + +* ``dfhack.with_suspend(f[,args...])`` + + Calls ``f`` with arguments after grabbing the DF core suspend lock. + Suspending is necessary for accessing a consistent state of DF memory. + + Returned values and errors are propagated through after releasing + the lock. It is safe to nest suspends. + + Every thread is allowed only one suspend per DF frame, so it is best + to group operations together in one big critical section. A plugin + can choose to run all lua code inside a C++-side suspend lock. + +* ``dfhack.call_with_finalizer(num_cleanup_args,always,cleanup_fn[,cleanup_args...],fn[,args...])`` + + Invokes ``fn`` with ``args``, and after it returns or throws an + error calls ``cleanup_fn`` with ``cleanup_args``. Any return values from + ``fn`` are propagated, and errors are re-thrown. + + The ``num_cleanup_args`` integer specifies the number of ``cleanup_args``, + and the ``always`` boolean specifies if cleanup should be called in any case, + or only in case of an error. + +* ``dfhack.with_finalize(cleanup_fn,fn[,args...])`` + + Calls ``fn`` with arguments, then finalizes with ``cleanup_fn``. + Implemented using ``call_with_finalizer(0,true,...)``. + +* ``dfhack.with_onerror(cleanup_fn,fn[,args...])`` + + Calls ``fn`` with arguments, then finalizes with ``cleanup_fn`` on any thrown error. + Implemented using ``call_with_finalizer(0,false,...)``. + +* ``dfhack.with_temp_object(obj,fn[,args...])`` + + Calls ``fn(obj,args...)``, then finalizes with ``obj:delete()``. + + Persistent configuration storage -------------------------------- @@ -1312,6 +1322,42 @@ Features: Invokes all listeners contained in the event in an arbitrary order using ``dfhack.safecall``. + +======= +Modules +======= + +DFHack sets up the lua interpreter so that the built-in ``require`` +function can be used to load shared lua code from hack/lua/. +The ``dfhack`` namespace reference itself may be obtained via +``require('dfhack')``, although it is initially created as a +global by C++ bootstrap code. + +The following functions are provided: + +* ``mkmodule(name)`` + + Creates an environment table for the module. Intended to be used as:: + + local _ENV = mkmodule('foo') + ... + return _ENV + + If called the second time, returns the same table; thus providing reload support. + +* ``reload(name)`` + + Reloads a previously ``require``-d module *"name"* from the file. + Intended as a help for module development. + +* ``dfhack.BASE_G`` + + This variable contains the root global environment table, which is + used as a base for all module and script environments. Its contents + should be kept limited to the standard Lua library and API described + in this document. + + ======= Plugins ======= @@ -1373,3 +1419,40 @@ sort Does not export any native functions as of now. Instead, it calls lua code to perform the actual ordering of list items. + + +======= +Scripts +======= + +Any files with the .lua extension placed into hack/scripts/* +are automatically used by the DFHack core as commands. The +matching command name consists of the name of the file sans +the extension. + +**NOTE:** Scripts placed in subdirectories still can be accessed, but +do not clutter the ``ls`` command list; thus it is preferred +for obscure developer-oriented scripts and scripts used by tools. +When calling such scripts, always use '/' as the separator for +directories, e.g. ``devel/lua-example``. + +Scripts are re-read from disk every time they are used +(this may be changed later to check the file change time); however +the global variable values persist in memory between calls. +Every script gets its own separate environment for global +variables. + +Arguments are passed in to the scripts via the **...** built-in +quasi-variable; when the script is called by the DFHack core, +they are all guaranteed to be non-nil strings. + +DFHack core invokes the scripts in the *core context* (see above); +however it is possible to call them from any lua code (including +from other scripts) in any context, via the same function the core uses: + +* ``dfhack.run_script(name[,args...])`` + + Run a lua script in hack/scripts/, as if it was started from dfhack command-line. + The ``name`` argument should be the name stem, as would be used on the command line. + +Note that this function lets errors propagate to the caller. diff --git a/Lua API.html b/Lua API.html index 47cf08ab6..04e899366 100644 --- a/Lua API.html +++ b/Lua API.html @@ -320,7 +320,7 @@ ul.auto-toc { -
      -

      DF structure wrapper

      +

      The current version of DFHack has extensive support for +the Lua scripting language, providing access to:

      +
        +
      1. Raw data structures used by the game.
        2. +
      2. Many C++ functions for high-level access to these +structures, and interaction with dfhack itself.
        3. +
      3. Some functions exported by C++ plugins.
        4. +
      +

      Lua code can be used both for writing scripts, which +are treated by DFHack command line prompt almost as +native C++ commands, and invoked by plugins written in c++.

      +

      This document describes native API available to Lua in detail. +For the most part it does not describe utility functions +implemented by Lua files located in hack/lua/...

      +
      +

      DF data structure wrapper

      DF structures described by the xml files in library/xml are exported to lua code as a tree of objects and functions under the df global, which broadly maps to the df namespace in C++.

      @@ -764,28 +780,8 @@ string, global environment and command-line history file.

    -
    -

    Miscellaneous

    -
      -

    • dfhack.run_script(name[,args...])

      -

      Run a lua script in hack/scripts/, as if it was started from dfhack command-line. -The name argument should be the name stem, as would be used on the command line. -Note that the script is re-read from the file every time it is called, and errors -are propagated to the caller.

      -
      • -

    • dfhack.with_suspend(f[,args...])

      -

      Calls f with arguments after grabbing the DF core suspend lock. -Suspending is necessary for accessing a consistent state of DF memory.

      -

      Returned values and errors are propagated through after releasing -the lock. It is safe to nest suspends.

      -

      Every thread is allowed only one suspend per DF frame, so it is best -to group operations together in one big critical section. A plugin -can choose to run all lua code inside a C++-side suspend lock.

      -
      • -
    -
    -

    Exception handling

    +

    Exception handling

    • dfhack.error(msg[,level[,verbose]])

      Throws a dfhack exception object with location and stack trace. @@ -808,25 +804,6 @@ returning. Intended as a convenience function.

    • dfhack.saferesume(coroutine[,args...])

      Compares to coroutine.resume like dfhack.safecall vs pcall.

      • -

    • dfhack.call_with_finalizer(num_cleanup_args,always,cleanup_fn[,cleanup_args...],fn[,args...])

      -

      Invokes fn with args, and after it returns or throws an -error calls cleanup_fn with cleanup_args. Any return values from -fn are propagated, and errors are re-thrown.

      -

      The num_cleanup_args integer specifies the number of cleanup_args, -and the always boolean specifies if cleanup should be called in any case, -or only in case of an error.

      -
      • -

    • dfhack.with_finalize(cleanup_fn,fn[,args...])

      -

      Calls fn with arguments, then finalizes with cleanup_fn. -Implemented using call_with_finalizer(0,true,...).

      -
      • -

    • dfhack.with_onerror(cleanup_fn,fn[,args...])

      -

      Calls fn with arguments, then finalizes with cleanup_fn on any thrown error. -Implemented using call_with_finalizer(0,false,...).

      -
      • -

    • dfhack.with_temp_object(obj,fn[,args...])

      -

      Calls fn(obj,args...), then finalizes with obj:delete().

      -

    • dfhack.exception

      Metatable of error objects used by dfhack. The objects have the following properties:

      @@ -859,6 +836,39 @@ following properties:

    +
    +

    Locking and finalization

    +
      +

    • dfhack.with_suspend(f[,args...])

      +

      Calls f with arguments after grabbing the DF core suspend lock. +Suspending is necessary for accessing a consistent state of DF memory.

      +

      Returned values and errors are propagated through after releasing +the lock. It is safe to nest suspends.

      +

      Every thread is allowed only one suspend per DF frame, so it is best +to group operations together in one big critical section. A plugin +can choose to run all lua code inside a C++-side suspend lock.

      +
      • +

    • dfhack.call_with_finalizer(num_cleanup_args,always,cleanup_fn[,cleanup_args...],fn[,args...])

      +

      Invokes fn with args, and after it returns or throws an +error calls cleanup_fn with cleanup_args. Any return values from +fn are propagated, and errors are re-thrown.

      +

      The num_cleanup_args integer specifies the number of cleanup_args, +and the always boolean specifies if cleanup should be called in any case, +or only in case of an error.

      +
      • +

    • dfhack.with_finalize(cleanup_fn,fn[,args...])

      +

      Calls fn with arguments, then finalizes with cleanup_fn. +Implemented using call_with_finalizer(0,true,...).

      +
      • +

    • dfhack.with_onerror(cleanup_fn,fn[,args...])

      +

      Calls fn with arguments, then finalizes with cleanup_fn on any thrown error. +Implemented using call_with_finalizer(0,false,...).

      +
      • +

    • dfhack.with_temp_object(obj,fn[,args...])

      +

      Calls fn(obj,args...), then finalizes with obj:delete().

      +
      • +
    +

    Persistent configuration storage

    This api is intended for storing configuration options in the world itself. @@ -1470,15 +1480,45 @@ order using dfhack.safecall.

    +
    +

    Modules

    +

    DFHack sets up the lua interpreter so that the built-in require +function can be used to load shared lua code from hack/lua/. +The dfhack namespace reference itself may be obtained via +require('dfhack'), although it is initially created as a +global by C++ bootstrap code.

    +

    The following functions are provided:

    +
      +

    • mkmodule(name)

      +

      Creates an environment table for the module. Intended to be used as:

      +
      +local _ENV = mkmodule('foo')
+...
+return _ENV
+
      +

      If called the second time, returns the same table; thus providing reload support.

      +
      • +

    • reload(name)

      +

      Reloads a previously require-d module "name" from the file. +Intended as a help for module development.

      +
      • +

    • dfhack.BASE_G

      +

      This variable contains the root global environment table, which is +used as a base for all module and script environments. Its contents +should be kept limited to the standard Lua library and API described +in this document.

      +
      • +
    +
    -

    Plugins

    +

    Plugins

    DFHack plugins may export native functions and events to lua contexts. They are automatically imported by mkmodule('plugins.<name>'); this means that a lua module file is still necessary for require to read.

    The following plugins have lua support.

    -

    burrows

    +

    burrows

    Implements extended burrow manipulations.

    Events:

      @@ -1516,11 +1556,41 @@ set is the same as used by the command line.

      The lua module file also re-exports functions from dfhack.burrows.

    -

    sort

    +

    sort

    Does not export any native functions as of now. Instead, it calls lua code to perform the actual ordering of list items.

    +
    +

    Scripts

    +

    Any files with the .lua extension placed into hack/scripts/* +are automatically used by the DFHack core as commands. The +matching command name consists of the name of the file sans +the extension.

    +

    NOTE: Scripts placed in subdirectories still can be accessed, but +do not clutter the ls command list; thus it is preferred +for obscure developer-oriented scripts and scripts used by tools. +When calling such scripts, always use '/' as the separator for +directories, e.g. devel/lua-example.

    +

    Scripts are re-read from disk every time they are used +(this may be changed later to check the file change time); however +the global variable values persist in memory between calls. +Every script gets its own separate environment for global +variables.

    +

    Arguments are passed in to the scripts via the ... built-in +quasi-variable; when the script is called by the DFHack core, +they are all guaranteed to be non-nil strings.

    +

    DFHack core invokes the scripts in the core context (see above); +however it is possible to call them from any lua code (including +from other scripts) in any context, via the same function the core uses:

    +
      +

    • dfhack.run_script(name[,args...])

      +

      Run a lua script in hack/scripts/, as if it was started from dfhack command-line. +The name argument should be the name stem, as would be used on the command line.

      +
      • +
    +

    Note that this function lets errors propagate to the caller.

    +
    diff --git a/library/lua/dfhack.lua b/library/lua/dfhack.lua index d200a6c5c..d56d4df60 100644 --- a/library/lua/dfhack.lua +++ b/library/lua/dfhack.lua @@ -68,6 +68,8 @@ function dfhack.with_temp_object(obj,fn,...) return dfhack.call_with_finalizer(1,true,call_delete,obj,fn,obj,...) end +dfhack.exception.__index = dfhack.exception + -- Module loading function mkmodule(module,env) From 0e582901ee972853999dfc86dbe93093a65cd35b Mon Sep 17 00:00:00 2001 From: Alexander Gavrilov Date: Fri, 22 Jun 2012 20:41:15 +0400 Subject: [PATCH 28/28] Update the compile document with some info about df-structures and RPC. --- COMPILE.rst | 23 ++++++++++++++++++++++- Compile.html | 33 ++++++++++++++++++++++++--------- Readme.html | 7 ++++++- 3 files changed, 52 insertions(+), 11 deletions(-) diff --git a/COMPILE.rst b/COMPILE.rst index beb738363..8fca0e1f5 100644 --- a/COMPILE.rst +++ b/COMPILE.rst @@ -152,10 +152,13 @@ Valid and useful build types include 'Release', 'Debug' and ================================ Using the library as a developer ================================ -Currently, the only way to use the library is to write a plugin that can be loaded by it. + +Currently, the most direct way to use the library is to write a plugin that can be loaded by it. All the plugins can be found in the 'plugins' folder. There's no in-depth documentation on how to write one yet, but it should be easy enough to copy one and just follow the pattern. +Other than through plugins, it is possible to use DFHack via remote access interface, or by writing Lua scripts. + The most important parts of DFHack are the Core, Console, Modules and Plugins. * Core acts as the centerpiece of DFHack - it acts as a filter between DF and SDL and synchronizes the various plugins with DF. @@ -171,6 +174,24 @@ The main license is zlib/libpng, some bits are MIT licensed, and some are BSD li Feel free to add your own extensions and plugins. Contributing back to the dfhack repository is welcome and the right thing to do :) +DF data structure definitions +============================= + +DFHack uses information about the game data structures, represented via xml files in the library/xml/ submodule. + +Data structure layouts are described in files following the df.*.xml name pattern. This information is transformed by a perl script into C++ headers describing the structures, and associated metadata for the Lua wrapper. These headers and data are then compiled into the DFHack libraries, thus necessitating a compatibility break every time layouts change; in return it significantly boosts the efficiency and capabilities of DFHack code. + +Global object addresses are stored in symbols.xml, which is copied to the dfhack release package and loaded as data at runtime. + +Remote access interface +======================= + +DFHack supports remote access by exchanging Google protobuf messages via a TCP socket. Both the core and plugins can define remotely accessible methods. The ``dfhack-run`` command uses this interface to invoke ordinary console commands. + +Currently the supported set of requests is limited, because the developers don't know what exactly is most useful. + +Protocol client implementations exist for Java and C#. + Contributing to DFHack ====================== diff --git a/Compile.html b/Compile.html index b0f9e9c6e..e17e57e22 100644 --- a/Compile.html +++ b/Compile.html @@ -334,10 +334,12 @@ ul.auto-toc {
  • Build types
  • Using the library as a developer
      -
    • Contributing to DFHack @@ -470,9 +472,10 @@ cmake .. -DCMAKE_BUILD_TYPE:string=BUILD_TYPE

      Using the library as a developer

      -

      Currently, the only way to use the library is to write a plugin that can be loaded by it. +

      Currently, the most direct way to use the library is to write a plugin that can be loaded by it. All the plugins can be found in the 'plugins' folder. There's no in-depth documentation on how to write one yet, but it should be easy enough to copy one and just follow the pattern.

      +

      Other than through plugins, it is possible to use DFHack via remote access interface, or by writing Lua scripts.

      The most important parts of DFHack are the Core, Console, Modules and Plugins.

      • Core acts as the centerpiece of DFHack - it acts as a filter between DF and SDL and synchronizes the various plugins with DF.
        • @@ -485,18 +488,30 @@ on how to write one yet, but it should be easy enough to copy one and just follo The main license is zlib/libpng, some bits are MIT licensed, and some are BSD licensed.

        Feel free to add your own extensions and plugins. Contributing back to the dfhack repository is welcome and the right thing to do :)

        +
        +

        DF data structure definitions

        +

        DFHack uses information about the game data structures, represented via xml files in the library/xml/ submodule.

        +

        Data structure layouts are described in files following the df.*.xml name pattern. This information is transformed by a perl script into C++ headers describing the structures, and associated metadata for the Lua wrapper. These headers and data are then compiled into the DFHack libraries, thus necessitating a compatibility break every time layouts change; in return it significantly boosts the efficiency and capabilities of DFHack code.

        +

        Global object addresses are stored in symbols.xml, which is copied to the dfhack release package and loaded as data at runtime.

        +
        +
        +

        Remote access interface

        +

        DFHack supports remote access by exchanging Google protobuf messages via a TCP socket. Both the core and plugins can define remotely accessible methods. The dfhack-run command uses this interface to invoke ordinary console commands.

        +

        Currently the supported set of requests is limited, because the developers don't know what exactly is most useful.

        +

        Protocol client implementations exist for Java and C#.

        +
        -

        Contributing to DFHack

        +

        Contributing to DFHack

        Several things should be kept in mind when contributing to DFHack.

        -

        Coding style

        +

        Coding style

        DFhack uses ANSI formatting and four spaces as indentation. Line endings are UNIX. The files use UTF-8 encoding. Code not following this won't make me happy, because I'll have to fix it. There's a good chance I'll make you fix it ;)

        -

        How to get new code into DFHack

        +

        How to get new code into DFHack

        You can send patches or make a clone of the github repo and ask me on the IRC channel to pull your code in. I'll review it and see if there are any problems. I'll fix them if they are minor.

        @@ -506,7 +521,7 @@ this is also a good place to dump new ideas and/or bugs that need fixing.

        -

        Memory research

        +

        Memory research

        If you want to do memory research, you'll need some tools and some knowledge. In general, you'll need a good memory viewer and optionally something to look at machine code without getting crazy :)

        diff --git a/Readme.html b/Readme.html index cd579d374..50ceae999 100644 --- a/Readme.html +++ b/Readme.html @@ -1350,7 +1350,12 @@ produce undesirable results. There are a few good ones though.

        You are in fort game mode, managing your fortress and paused. You switch to the arena game mode, assume control of a creature and then switch to adventure game mode(1). -You just lost a fortress and gained an adventurer.

        +You just lost a fortress and gained an adventurer. +You could also do this. +You are in fort game mode, managing your fortress and paused at the esc menu. +You switch to the adventure game mode, then use Dfusion to assume control of a creature and then +save or retire. +You just created a returnable mountain home and gained an adventurer.

        I take no responsibility of anything that happens as a result of using this tool