Merge https://github.com/warmist/dfhack

LUA_API.rst

@@ -799,3 +799,17 @@ Maps module
 * ``dfhack.maps.setBlockBurrowTile(burrow,block,x,y,enable)``
 
   Adds or removes the tile from the burrow. Returns *false* if invalid coords.
+
+
+Core interpreter context
+========================
+
+While plugins can create any number of interpreter instances,
+there is one special context managed by dfhack core. It is the
+only context that can receive events from DF and plugins.
+
+Core context specific functions:
+
+* ``dfhack.is_core_context``
+
+  Boolean value; *true* in the core context.

Lua API.html

@@ -344,6 +344,7 @@ ul.auto-toc {
 <li><a class="reference internal" href="#maps-module" id="id18">Maps module</a></li>
 </ul>
 </li>
+<li><a class="reference internal" href="#core-interpreter-context" id="id19">Core interpreter context</a></li>
 </ul>
 </li>
 </ul>
@@ -1010,6 +1011,18 @@ Returns <em>false</em> in case of error.</p>
 </ul>
 </div>
 </div>
+<div class="section" id="core-interpreter-context">
+<h2><a class="toc-backref" href="#id19">Core interpreter context</a></h2>
+<p>While plugins can create any number of interpreter instances,
+there is one special context managed by dfhack core. It is the
+only context that can receive events from DF and plugins.</p>
+<p>Core context specific functions:</p>
+<ul>
+<li><p class="first"><tt class="docutils literal">dfhack.is_core_context</tt></p>
+<p>Boolean value; <em>true</em> in the core context.</p>
+</li>
+</ul>
+</div>
 </div>
 </div>
 </body>

build/generate-MSVC-all-breakfast.bat

@@ -4,5 +4,6 @@ IF NOT EXIST DF_PATH.txt SET _DF_PATH=%CD%\DF
 mkdir VC2010
 cd VC2010
 echo generating a build folder
-for /f "delims=" %%a in ('DATE /T') do @set myvar=breakfast-%%a 
+rem for /f "delims=" %%a in ('DATE /T') do @set myvar=breakfast-%BUILD_NUMBER%
+set myvar=breakfast-%BUILD_NUMBER%
 cmake ..\.. -G"Visual Studio 10" -DDFHACK_RELEASE="%myvar%" -DCMAKE_INSTALL_PREFIX="%_DF_PATH%" -DBUILD_DEVEL=1 -DBUILD_DEV_PLUGINS=1 -DBUILD_DF2MC=1 -DBUILD_DFUSION=1 -DBUILD_STONESENSE=1 -DBUILD_SERVER=1

build/package-debug.bat

@@ -1,4 +1,6 @@
+@echo off
 call "%VS100COMNTOOLS%vsvars32.bat"
 cd VC2010
 msbuild /m /p:Platform=Win32 /p:Configuration=RelWithDebInfo PACKAGE.vcxproj
 cd ..
+exit %ERRORLEVEL%

build/package-release.bat

@@ -1,4 +1,6 @@
+@echo off
 call "%VS100COMNTOOLS%vsvars32.bat"
 cd VC2010
 msbuild /m /p:Platform=Win32 /p:Configuration=Release PACKAGE.vcxproj
 cd ..
+exit %ERRORLEVEL%

depends/lua/CMakeLists.txt

@@ -1,7 +1,7 @@
 PROJECT ( lua CXX )
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 
-SET(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-DLUA_USE_APICHECK")
+SET(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -DLUA_USE_APICHECK")
 
 IF(WIN32)
     ADD_DEFINITIONS(-D_CRT_SECURE_NO_DEPRECATE )

depends/lua/src/ldo.c

@@ -403,7 +403,7 @@ static void finishCcall (lua_State *L) {
   lua_assert(ci->u.c.k != NULL);  /* must have a continuation */
   lua_assert(L->nny == 0);
   /* finish 'luaD_call' */
-  L->nCcalls--;
+  //L->nCcalls--;
   /* finish 'lua_callk' */
   adjustresults(L, ci->nresults);
   /* call continuation function */
@@ -513,7 +513,7 @@ static void resume (lua_State *L, void *ud) {
         api_checknelems(L, n);
         firstArg = L->top - n;  /* yield results come from continuation */
       }
-      L->nCcalls--;  /* finish 'luaD_call' */
+      //L->nCcalls--;  /* finish 'luaD_call' */
       luaD_poscall(L, firstArg);  /* finish 'luaD_precall' */
     }
     unroll(L, NULL);

library/Core.cpp

@@ -49,6 +49,8 @@ using namespace std;
 #include "modules/Graphic.h"
 #include "modules/Windows.h"
 #include "RemoteServer.h"
+#include "LuaTools.h"
+
 using namespace DFHack;
 
 #include "df/ui.h"
@@ -701,6 +703,9 @@ bool Core::Init()
     virtual_identity::Init(this);
     df::global::InitGlobals();
 
+    // initialize common lua context
+    Lua::Core::Init(con);
+
     // create mutex for syncing with interactive tasks
     misc_data_mutex=new mutex();
     cerr << "Initializing Plugins.\n";
@@ -803,6 +808,13 @@ void *Core::GetData( std::string key )
     }
 }
 
+bool Core::isSuspended(void)
+{
+    lock_guard<mutex> lock(d->AccessMutex);
+
+    return (d->df_suspend_depth > 0 && d->df_suspend_thread == this_thread::get_id());
+}
+
 void Core::Suspend()
 {
     auto tid = this_thread::get_id();
@@ -882,10 +894,13 @@ int Core::Update()
         Init();
         if(errorstate)
             return -1;
+        Lua::Core::Reset(con, "core init");
     }
 
     color_ostream_proxy out(con);
 
+    Lua::Core::Reset(out, "DF code execution");
+
     if (first_update)
         plug_mgr->OnStateChange(out, SC_CORE_INITIALIZED);
 
@@ -976,6 +991,8 @@ int Core::Update()
         assert(d->df_suspend_depth == 0);
         // destroy condition
         delete nc;
+        // check lua stack depth
+        Lua::Core::Reset(con, "suspend");
     }
 
     return 0;

library/LuaApi.cpp

@@ -74,30 +74,7 @@ distribution.
 using namespace DFHack;
 using namespace DFHack::LuaWrapper;
 
-template<class T>
+int Lua::PushPosXYZ(lua_State *state, df::coord pos)
-void push_pointer_vector(lua_State *state, const std::vector<T*> &pvec)
-{
-    lua_createtable(state,pvec.size(),0);
-
-    for (size_t i = 0; i < pvec.size(); i++)
-    {
-        Lua::PushDFObject(state, pvec[i]);
-        lua_rawseti(state, -2, i+1);
-    }
-}
-
-template<class T>
-T *get_checked_arg(lua_State *state, int arg)
-{
-    luaL_checkany(state, arg);
-
-    auto ptr = Lua::GetDFObject<T>(state, arg);
-    if (!ptr)
-        luaL_argerror(state, arg, "invalid type");
-    return ptr;
-}
-
-int push_pos(lua_State *state, df::coord pos)
 {
     if (!pos.isValid())
     {
@@ -570,9 +547,9 @@ static void OpenModule(lua_State *state, const char *mname,
     lua_pop(state, 1);
 }
 
-#define WRAPM(module, function) { #function, df::wrap_function(module::function) }
+#define WRAPM(module, function) { #function, df::wrap_function(module::function,true) }
-#define WRAP(function) { #function, df::wrap_function(function) }
+#define WRAP(function) { #function, df::wrap_function(function,true) }
-#define WRAPN(name, function) { #name, df::wrap_function(function) }
+#define WRAPN(name, function) { #name, df::wrap_function(function,true) }
 
 static const LuaWrapper::FunctionReg dfhack_module[] = {
     WRAPM(Translation, TranslateName),
@@ -613,7 +590,7 @@ static int job_listNewlyCreated(lua_State *state)
     if (Job::listNewlyCreated(&pvec, &nxid))
     {
         lua_pushinteger(state, nxid);
-        push_pointer_vector(state, pvec);
+        Lua::PushVector(state, pvec);
         return 2;
     }
     else
@@ -642,7 +619,7 @@ static const LuaWrapper::FunctionReg dfhack_units_module[] = {
 
 static int units_getPosition(lua_State *state)
 {
-    return push_pos(state, Units::getPosition(get_checked_arg<df::unit>(state,1)));
+    return Lua::PushPosXYZ(state, Units::getPosition(Lua::CheckDFObject<df::unit>(state,1)));
 }
 
 static const luaL_Reg dfhack_units_funcs[] = {
@@ -673,14 +650,14 @@ static const LuaWrapper::FunctionReg dfhack_items_module[] = {
 
 static int items_getPosition(lua_State *state)
 {
-    return push_pos(state, Items::getPosition(get_checked_arg<df::item>(state,1)));
+    return Lua::PushPosXYZ(state, Items::getPosition(Lua::CheckDFObject<df::item>(state,1)));
 }
 
 static int items_getContainedItems(lua_State *state)
 {
     std::vector<df::item*> pvec;
-    Items::getContainedItems(get_checked_arg<df::item>(state,1),&pvec);
+    Items::getContainedItems(Lua::CheckDFObject<df::item>(state,1),&pvec);
-    push_pointer_vector(state, pvec);
+    Lua::PushVector(state, pvec);
     return 1;
 }
 
@@ -719,8 +696,8 @@ static const LuaWrapper::FunctionReg dfhack_maps_module[] = {
 static int maps_listBurrowBlocks(lua_State *state)
 {
     std::vector<df::map_block*> pvec;
-    Maps::listBurrowBlocks(&pvec, get_checked_arg<df::burrow>(state,1));
+    Maps::listBurrowBlocks(&pvec, Lua::CheckDFObject<df::burrow>(state,1));
-    push_pointer_vector(state, pvec);
+    Lua::PushVector(state, pvec);
     return 1;
 }
 

library/LuaTools.cpp

@@ -58,9 +58,22 @@ distribution.
 #include <lauxlib.h>
 #include <lualib.h>
 
+#include <lstate.h>
+
 using namespace DFHack;
 using namespace DFHack::LuaWrapper;
 
+lua_State *DFHack::Lua::Core::State = NULL;
+
+inline void AssertCoreSuspend(lua_State *state)
+{
+    assert(!Lua::IsCoreContext(state) || DFHack::Core::getInstance().isSuspended());
+}
+
+/*
+ * Public DF object reference handling API
+ */
+
 void DFHack::Lua::PushDFObject(lua_State *state, type_identity *type, void *ptr)
 {
     push_object_internal(state, type, ptr, false);
@@ -71,6 +84,40 @@ void *DFHack::Lua::GetDFObject(lua_State *state, type_identity *type, int val_in
     return get_object_internal(state, type, val_index, exact_type, false);
 }
 
+void *DFHack::Lua::CheckDFObject(lua_State *state, type_identity *type, int val_index, bool exact_type)
+{
+    if (lua_type(state, val_index) == LUA_TNONE)
+    {
+        if (val_index > 0)
+            luaL_argerror(state, val_index, "pointer expected");
+        else
+            luaL_error(state, "at index %d: pointer expected", val_index);
+    }
+
+    if (lua_isnil(state, val_index))
+        return NULL;
+
+    void *rv = get_object_internal(state, type, val_index, exact_type, false);
+
+    if (!rv)
+    {
+        std::string error = "invalid pointer type";
+        if (type)
+            error += "; expected: " + type->getFullName();
+
+        if (val_index > 0)
+            luaL_argerror(state, val_index, error.c_str());
+        else
+            luaL_error(state, "at index %d: %s", val_index, error.c_str());
+    }
+
+    return rv;
+}
+
+/*
+ * Console I/O wrappers
+ */
+
 static int DFHACK_OSTREAM_TOKEN = 0;
 
 color_ostream *DFHack::Lua::GetOutput(lua_State *L)
@@ -193,15 +240,14 @@ static int lua_dfhack_is_interactive(lua_State *S)
     return 1;
 }
 
-static int lua_dfhack_lineedit(lua_State *S)
+static int dfhack_lineedit_sync(lua_State *S, Console *pstream)
 {
-    const char *prompt = luaL_optstring(S, 1, ">> ");
-    const char *hfile = luaL_optstring(S, 2, NULL);
-
-    Console *pstream = get_console(S);
     if (!pstream)
         return 2;
 
+    const char *prompt = luaL_optstring(S, 1, ">> ");
+    const char *hfile = luaL_optstring(S, 2, NULL);
+
     DFHack::CommandHistory hist;
     if (hfile)
         hist.load(hfile);
@@ -224,6 +270,51 @@ static int lua_dfhack_lineedit(lua_State *S)
     }
 }
 
+static int DFHACK_QUERY_COROTABLE_TOKEN = 0;
+
+static int yield_helper(lua_State *S)
+{
+    return lua_yield(S, lua_gettop(S));
+}
+
+namespace {
+    int dfhack_lineedit_cont(lua_State *L, int status, int)
+    {
+        if (Lua::IsSuccess(status))
+            return lua_gettop(L) - 2;
+        else
+            return dfhack_lineedit_sync(L, get_console(L));
+    }
+}
+
+static int dfhack_lineedit(lua_State *S)
+{
+    lua_settop(S, 2);
+
+    Console *pstream = get_console(S);
+    if (!pstream)
+        return 2;
+
+    lua_rawgetp(S, LUA_REGISTRYINDEX, &DFHACK_QUERY_COROTABLE_TOKEN);
+    lua_rawgetp(S, -1, S);
+    bool in_coroutine = !lua_isnil(S, -1);
+    lua_settop(S, 2);
+
+    if (in_coroutine)
+    {
+        lua_pushcfunction(S, yield_helper);
+        lua_pushvalue(S, 1);
+        lua_pushvalue(S, 2);
+        return Lua::TailPCallK<dfhack_lineedit_cont>(S, 2, LUA_MULTRET, 0, 0);
+    }
+
+    return dfhack_lineedit_sync(S, pstream);
+}
+
+/*
+ * Exception handling
+ */
+
 static int DFHACK_EXCEPTION_META_TOKEN = 0;
 
 static void error_tostring(lua_State *L, bool keep_old = false)
@@ -265,8 +356,21 @@ static void report_error(lua_State *L, color_ostream *out = NULL)
     lua_pop(L, 1);
 }
 
-static bool convert_to_exception(lua_State *L)
+static bool convert_to_exception(lua_State *L, int slevel, lua_State *thread = NULL)
 {
+    if (!thread)
+        thread = L;
+
+    if (thread == L)
+        lua_pushthread(L);
+    else
+    {
+        lua_pushthread(thread);
+        lua_xmove(thread, L, 1);
+    }
+
+    lua_swap(L);
+
     int base = lua_gettop(L);
 
     bool force_unknown = false;
@@ -277,13 +381,33 @@ static bool convert_to_exception(lua_State *L)
         bool is_exception = lua_rawequal(L, -1, -2);
         lua_settop(L, base);
 
-        // If it is an exception, return as is
         if (is_exception)
+        {
+            // If it is an exception from the same thread, return as is
+            lua_getfield(L, base, "thread");
+            bool same_thread = lua_rawequal(L, -1, base-1);
+            lua_settop(L, base);
+
+            if (same_thread)
+            {
+                lua_remove(L, base-1);
                 return false;
+            }
 
+            // Create a new exception for this thread
+            lua_newtable(L);
+            luaL_where(L, 1);
+            lua_pushstring(L, "coroutine resume failed");
+            lua_concat(L, 2);
+            lua_setfield(L, -2, "message");
+            lua_swap(L);
+            lua_setfield(L, -2, "cause");
+        }
+        else
             force_unknown = true;
     }
 
+    // Promote non-table to table, and do some sanity checks
     if (!lua_istable(L, base) || force_unknown)
     {
         lua_newtable(L);
@@ -313,6 +437,10 @@ static bool convert_to_exception(lua_State *L)
 
     lua_rawgetp(L, LUA_REGISTRYINDEX, &DFHACK_EXCEPTION_META_TOKEN);
     lua_setmetatable(L, base);
+    lua_swap(L);
+    lua_setfield(L, -2, "thread");
+    luaL_traceback(L, thread, NULL, slevel);
+    lua_setfield(L, -2, "stacktrace");
     return true;
 }
 
@@ -321,13 +449,7 @@ static int dfhack_onerror(lua_State *L)
     luaL_checkany(L, 1);
     lua_settop(L, 1);
 
-    bool changed = convert_to_exception(L);
+    convert_to_exception(L, 1);
-    if (!changed)
-        return 1;
-
-    luaL_traceback(L, L, NULL, 1);
-    lua_setfield(L, 1, "stacktrace");
-
     return 1;
 }
 
@@ -364,14 +486,8 @@ static void push_simple_error(lua_State *L, const char *str)
 {
     lua_pushstring(L, str);
 
-    if (lua_checkstack(L, 5))
-        convert_to_exception(L);
-
     if (lua_checkstack(L, LUA_MINSTACK))
-    {
+        convert_to_exception(L, 0);
-        luaL_traceback(L, L, NULL, 1);
-        lua_setfield(L, -2, "stacktrace");
-    }
 }
 
 static bool do_finish_pcall(lua_State *L, bool success, int base = 1, int space = 2)
@@ -387,37 +503,34 @@ static bool do_finish_pcall(lua_State *L, bool success, int base = 1, int space
     {
         lua_pushboolean(L, success);
         lua_replace(L, base); /* put first result in first slot */
-        return true;
+        return success;
     }
 }
 
-static int finish_dfhack_safecall (lua_State *L, bool success)
+namespace {
+    int safecall_cont(lua_State *L, int status, int)
     {
-    success = do_finish_pcall(L, success);
+        bool success = do_finish_pcall(L, Lua::IsSuccess(status));
 
         if (!success)
             report_error(L);
 
         return lua_gettop(L);
     }
-
-static int safecall_cont (lua_State *L)
-{
-    int status = lua_getctx(L, NULL);
-    return finish_dfhack_safecall(L, (status == LUA_YIELD));
 }
 
-static int lua_dfhack_safecall (lua_State *L)
+static int dfhack_safecall (lua_State *L)
 {
     luaL_checkany(L, 1);
     lua_pushcfunction(L, dfhack_onerror);
     lua_insert(L, 1);
-    int status = lua_pcallk(L, lua_gettop(L) - 2, LUA_MULTRET, 1, 0, safecall_cont);
+    return Lua::TailPCallK<safecall_cont>(L, lua_gettop(L) - 2, LUA_MULTRET, 1, 0);
-    return finish_dfhack_safecall(L, (status == LUA_OK));
 }
 
 bool DFHack::Lua::SafeCall(color_ostream &out, lua_State *L, int nargs, int nres, bool perr)
 {
+    AssertCoreSuspend(L);
+
     int base = lua_gettop(L) - nargs;
 
     color_ostream *cur_out = Lua::GetOutput(L);
@@ -440,13 +553,180 @@ bool DFHack::Lua::SafeCall(color_ostream &out, lua_State *L, int nargs, int nres
     return ok;
 }
 
-bool DFHack::Lua::Require(color_ostream &out, lua_State *state,
+// Copied from lcorolib.c, with error handling modifications
-                          const std::string &module, bool setglobal)
+static int resume_helper(lua_State *L, lua_State *co, int narg, int nres)
+{
+    if (!co) {
+        lua_pop(L, narg);
+        push_simple_error(L, "coroutine expected in resume");
+        return LUA_ERRRUN;
+    }
+    if (!lua_checkstack(co, narg)) {
+        lua_pop(L, narg);
+        push_simple_error(L, "too many arguments to resume");
+        return LUA_ERRRUN;
+    }
+    if (lua_status(co) == LUA_OK && lua_gettop(co) == 0) {
+        lua_pop(L, narg);
+        push_simple_error(L, "cannot resume dead coroutine");
+        return LUA_ERRRUN;
+    }
+    lua_xmove(L, co, narg);
+    int status = lua_resume(co, L, narg);
+    if (Lua::IsSuccess(status))
+    {
+        int nact = lua_gettop(co);
+        if (nres == LUA_MULTRET)
+            nres = nact;
+        else if (nres < nact)
+            lua_settop(co, nact = nres);
+        if (!lua_checkstack(L, nres + 1)) {
+            lua_settop(co, 0);
+            push_simple_error(L, "too many results to resume");
+            return LUA_ERRRUN;
+        }
+        int ttop = lua_gettop(L) + nres;
+        lua_xmove(co, L, nact);
+        lua_settop(L, ttop);
+    }
+    else
     {
+        lua_xmove(co, L, 1);
+
+        // A cross-thread version of dfhack_onerror
+        if (lua_checkstack(L, LUA_MINSTACK))
+            convert_to_exception(L, 0, co);
+    }
+    return status;
+}
+
+static int dfhack_coresume (lua_State *L) {
+    lua_State *co = lua_tothread(L, 1);
+    luaL_argcheck(L, !!co, 1, "coroutine expected");
+    int r = resume_helper(L, co, lua_gettop(L) - 1, LUA_MULTRET);
+    bool ok = Lua::IsSuccess(r);
+    lua_pushboolean(L, ok);
+    lua_insert(L, 2);
+    return lua_gettop(L) - 1;
+}
+
+static int dfhack_saferesume (lua_State *L) {
+    lua_State *co = lua_tothread(L, 1);
+    luaL_argcheck(L, !!co, 1, "coroutine expected");
+    int r = resume_helper(L, co, lua_gettop(L) - 1, LUA_MULTRET);
+    bool ok = Lua::IsSuccess(r);
+    lua_pushboolean(L, ok);
+    lua_insert(L, 2);
+    if (!ok)
+        report_error(L);
+    return lua_gettop(L) - 1;
+}
+
+static int dfhack_coauxwrap (lua_State *L) {
+    lua_State *co = lua_tothread(L, lua_upvalueindex(1));
+    int r = resume_helper(L, co, lua_gettop(L), LUA_MULTRET);
+    if (Lua::IsSuccess(r))
+        return lua_gettop(L);
+    else
+        return lua_error(L);
+}
+
+static int dfhack_cowrap (lua_State *L) {
+    luaL_checktype(L, 1, LUA_TFUNCTION);
+    lua_settop(L, 1);
+    Lua::NewCoroutine(L);
+    lua_pushcclosure(L, dfhack_coauxwrap, 1);
+    return 1;
+}
+
+lua_State *DFHack::Lua::NewCoroutine(lua_State *L) {
+    lua_State *NL = lua_newthread(L);
+    lua_swap(L);
+    lua_xmove(L, NL, 1); /* move function from L to NL */
+    return NL;
+}
+
+int DFHack::Lua::SafeResume(color_ostream &out, lua_State *from, lua_State *thread, int nargs, int nres, bool perr)
+{
+    AssertCoreSuspend(from);
+
+    color_ostream *cur_out = Lua::GetOutput(from);
+    set_dfhack_output(from, &out);
+
+    int rv = resume_helper(from, thread, nargs, nres);
+
+    if (!Lua::IsSuccess(rv) && perr)
+    {
+        report_error(from, &out);
+        lua_pop(from, 1);
+    }
+
+    set_dfhack_output(from, cur_out);
+
+    return rv;
+}
+
+int DFHack::Lua::SafeResume(color_ostream &out, lua_State *from, int nargs, int nres, bool perr)
+{
+    int base = lua_gettop(from) - nargs;
+    lua_State *thread = lua_tothread(from, base);
+
+    int rv = SafeResume(out, from, thread, nargs, nres, perr);
+
+    lua_remove(from, base);
+    return rv;
+}
+
+/*
+ * Module loading
+ */
+
+static int DFHACK_LOADED_TOKEN = 0;
+
+bool DFHack::Lua::PushModule(color_ostream &out, lua_State *state, const char *module)
+{
+    AssertCoreSuspend(state);
+
+    // Check if it is already loaded
+    lua_rawgetp(state, LUA_REGISTRYINDEX, &DFHACK_LOADED_TOKEN);
+    lua_pushstring(state, module);
+    lua_rawget(state, -2);
+
+    if (lua_toboolean(state, -1))
+    {
+        lua_remove(state, -2);
+        return true;
+    }
+
+    lua_pop(state, 2);
     lua_getglobal(state, "require");
-    lua_pushstring(state, module.c_str());
+    lua_pushstring(state, module);
+
+    return Lua::SafeCall(out, state, 1, 1);
+}
+
+bool DFHack::Lua::PushModulePublic(color_ostream &out, lua_State *state,
+                                   const char *module, const char *name)
+{
+    if (!PushModule(out, state, module))
+        return false;
 
-    if (!Lua::SafeCall(out, state, 1, 1))
+    if (!lua_istable(state, -1))
+    {
+        lua_pop(state, 1);
+        return false;
+    }
+
+    lua_pushstring(state, name);
+    lua_rawget(state, -2);
+    lua_remove(state, -2);
+    return true;
+}
+
+bool DFHack::Lua::Require(color_ostream &out, lua_State *state,
+                          const std::string &module, bool setglobal)
+{
+    if (!PushModule(out, state, module.c_str()))
         return false;
 
     if (setglobal)
@@ -471,6 +751,8 @@ bool DFHack::Lua::SafeCallString(color_ostream &out, lua_State *state, const std
                                  int nargs, int nres, bool perr,
                                  const char *debug_tag, int env_idx)
 {
+    AssertCoreSuspend(state);
+
     if (!debug_tag)
         debug_tag = code.c_str();
     if (env_idx)
@@ -504,129 +786,144 @@ bool DFHack::Lua::SafeCallString(color_ostream &out, lua_State *state, const std
     return Lua::SafeCall(out, state, nargs, nres, perr);
 }
 
-bool DFHack::Lua::InterpreterLoop(color_ostream &out, lua_State *state,
+/*
-                                  const char *prompt, int env, const char *hfile)
+ * Coroutine interactive query loop
+ */
+
+static int resume_query_loop(color_ostream &out,
+                             lua_State *thread, lua_State *state, int nargs,
+                             std::string &prompt, std::string &histfile)
+{
+    int rv = Lua::SafeResume(out, state, thread, nargs, 2);
+
+    if (Lua::IsSuccess(rv))
+    {
+        prompt = ifnull(lua_tostring(state, -2), "");
+        histfile = ifnull(lua_tostring(state, -1), "");
+        lua_pop(state, 2);
+    }
+
+    return rv;
+}
+
+bool DFHack::Lua::RunCoreQueryLoop(color_ostream &out, lua_State *state,
+                                   bool (*init)(color_ostream&, lua_State*, void*),
+                                   void *arg)
 {
     if (!out.is_console())
         return false;
     if (!lua_checkstack(state, 20))
         return false;
 
-    if (!hfile)
+    Console &con = static_cast<Console&>(out);
-        hfile = "lua.history";
-    if (!prompt)
-        prompt = "lua";
-
-    DFHack::CommandHistory hist;
-    hist.load(hfile);
 
-    out.print("Type quit to exit interactive lua interpreter.\n");
+    lua_State *thread;
+    int rv;
+    std::string prompt;
+    std::string histfile;
 
-    static bool print_banner = true;
+    DFHack::CommandHistory hist;
-    if (print_banner) {
+    std::string histname;
-        out.print("Shortcuts:\n"
-                  " '= foo' => '_1,_2,... = foo'\n"
-                  " '! foo' => 'print(foo)'\n"
-                  "Both save the first result as '_'.\n");
-        print_banner = false;
-    }
 
-    Console &con = static_cast<Console&>(out);
+    {
+        CoreSuspender suspend;
 
-    // Make a proxy global environment.
-    lua_newtable(state);
         int base = lua_gettop(state);
 
-    lua_newtable(state);
+        if (!init(out, state, arg))
-    if (env)
+        {
-        lua_pushvalue(state, env);
-    else
-        lua_rawgeti(state, LUA_REGISTRYINDEX, LUA_RIDX_GLOBALS);
-    lua_setfield(state, -2, "__index");
-    lua_setmetatable(state, -2);
-
-    // Main interactive loop
-    int vcnt = 1;
-    string curline;
-    string prompt_str = "[" + string(prompt) + "]# ";
-
-    for (;;) {
             lua_settop(state, base);
+            return false;
+        }
 
-        con.lineedit(prompt_str,curline,hist);
+        lua_rawgetp(state, LUA_REGISTRYINDEX, &DFHACK_QUERY_COROTABLE_TOKEN);
+        lua_pushvalue(state, base+1);
+        lua_remove(state, base+1);
+        thread = Lua::NewCoroutine(state);
+        lua_rawsetp(state, -2, thread);
+        lua_pop(state, 1);
 
-        if (curline.empty())
+        rv = resume_query_loop(out, thread, state, lua_gettop(state)-base, prompt, histfile);
-            continue;
+    }
-        if (curline == "quit")
-            break;
 
-        hist.add(curline);
+    while (rv == LUA_YIELD)
+    {
+        if (histfile != histname)
+        {
+            if (!histname.empty())
+                hist.save(histname.c_str());
 
-        char pfix = curline[0];
+            hist.clear();
+            histname = histfile;
 
-        if (pfix == '=' || pfix == '!')
+            if (!histname.empty())
-        {
+                hist.load(histname.c_str());
-            curline = "return " + curline.substr(1);
+        }
 
-            if (!Lua::SafeCallString(out, state, curline, 0, LUA_MULTRET, true, "=(interactive)", base))
+        if (prompt.empty())
-                continue;
+            prompt = ">> ";
 
-            int numret = lua_gettop(state) - base;
+        std::string curline;
+        con.lineedit(prompt,curline,hist);
+        hist.add(curline);
 
-            if (numret >= 1)
         {
-                lua_pushvalue(state, base+1);
+            CoreSuspender suspend;
-                lua_setfield(state, base, "_");
 
-                if (pfix == '!')
+            lua_pushlstring(state, curline.data(), curline.size());
-                {
+            rv = resume_query_loop(out, thread, state, 1, prompt, histfile);
-                    lua_pushcfunction(state, lua_dfhack_println);
-                    lua_insert(state, base+1);
-                    SafeCall(out, state, numret, 0);
-                    continue;
         }
     }
 
-            for (int i = 1; i <= numret; i++)
+    if (!histname.empty())
-            {
+        hist.save(histname.c_str());
-                std::string name = stl_sprintf("_%d", vcnt++);
-                lua_pushvalue(state, base + i);
-                lua_setfield(state, base, name.c_str());
-
-                out.print("%s = ", name.c_str());
 
-                lua_pushcfunction(state, lua_dfhack_println);
-                lua_pushvalue(state, base + i);
-                SafeCall(out, state, 1, 0);
-            }
-        }
-        else
     {
-            if (!Lua::SafeCallString(out, state, curline, 0, LUA_MULTRET, true, "=(interactive)", base))
+        CoreSuspender suspend;
-                continue;
+
+        lua_rawgetp(state, LUA_REGISTRYINDEX, &DFHACK_QUERY_COROTABLE_TOKEN);
+        lua_pushnil(state);
+        lua_rawsetp(state, -2, thread);
+        lua_pop(state, 1);
     }
+
+    return (rv == LUA_OK);
 }
 
-    lua_settop(state, base-1);
+namespace {
+    struct InterpreterArgs {
+        const char *prompt;
+        const char *hfile;
+    };
+}
 
-    hist.save(hfile);
+static bool init_interpreter(color_ostream &out, lua_State *state, void *info)
+{
+    auto args = (InterpreterArgs*)info;
+    lua_getglobal(state, "dfhack");
+    lua_getfield(state, -1, "interpreter");
+    lua_remove(state, -2);
+    lua_pushstring(state, args->prompt);
+    lua_pushstring(state, args->hfile);
     return true;
 }
 
-static int lua_dfhack_interpreter(lua_State *state)
+bool DFHack::Lua::InterpreterLoop(color_ostream &out, lua_State *state,
+                                  const char *prompt, const char *hfile)
 {
-    Console *pstream = get_console(state);
+    if (!out.is_console())
-    if (!pstream)
+        return false;
-        return 2;
 
-    int argc = lua_gettop(state);
+    if (!hfile)
+        hfile = "lua.history";
+    if (!prompt)
+        prompt = "lua";
 
-    const char *prompt = (argc >= 1 ? lua_tostring(state, 1) : NULL);
+    InterpreterArgs args;
-    int env = (argc >= 2 && !lua_isnil(state,2) ? 2 : 0);
+    args.prompt = prompt;
-    const char *hfile = (argc >= 3 ? lua_tostring(state, 3) : NULL);
+    args.hfile = hfile;
 
-    lua_pushboolean(state, Lua::InterpreterLoop(*pstream, state, prompt, env, hfile));
+    return RunCoreQueryLoop(out, state, init_interpreter, &args);
-    return 1;
 }
 
 static bool do_invoke_cleanup(lua_State *L, int nargs, int errorfun, bool success)
@@ -648,8 +945,10 @@ static bool do_invoke_cleanup(lua_State *L, int nargs, int errorfun, bool succes
     return success;
 }
 
-static int finish_dfhack_cleanup (lua_State *L, bool success)
+int dfhack_cleanup_cont(lua_State *L, int status, int)
 {
+    bool success = Lua::IsSuccess(status);
+
     int nargs = lua_tointeger(L, 1);
     bool always = lua_toboolean(L, 2);
     int rvbase = 4+nargs;
@@ -690,12 +989,6 @@ static int finish_dfhack_cleanup (lua_State *L, bool success)
     return numret;
 }
 
-static int dfhack_cleanup_cont (lua_State *L)
-{
-    int status = lua_getctx(L, NULL);
-    return finish_dfhack_cleanup(L, (status == LUA_YIELD));
-}
-
 static int dfhack_call_with_finalizer (lua_State *L)
 {
     int nargs = luaL_checkint(L, 1);
@@ -730,8 +1023,7 @@ static int dfhack_call_with_finalizer (lua_State *L)
     // Actually invoke
 
     // stack: [nargs] [always] [errorfun] [cleanup fun] [cleanup args...] |rvbase+1:| [fun] [args...]
-    int status = lua_pcallk(L, lua_gettop(L)-rvbase-1, LUA_MULTRET, 3, 0, dfhack_cleanup_cont);
+    return Lua::TailPCallK<dfhack_cleanup_cont>(L, lua_gettop(L)-rvbase-1, LUA_MULTRET, 3, 0);
-    return finish_dfhack_cleanup(L, (status == LUA_OK));
 }
 
 static int lua_dfhack_with_suspend(lua_State *L)
@@ -761,15 +1053,24 @@ static int dfhack_open_plugin(lua_State *L)
     return 0;
 }
 
+bool Lua::IsCoreContext(lua_State *state)
+{
+    // This uses a private field of the lua state to
+    // evaluate the condition without accessing the lua
+    // stack, and thus requiring a lock on the core state.
+    return state && Lua::Core::State &&
+           state->l_G == Lua::Core::State->l_G;
+}
+
 static const luaL_Reg dfhack_funcs[] = {
     { "print", lua_dfhack_print },
     { "println", lua_dfhack_println },
     { "printerr", lua_dfhack_printerr },
     { "color", lua_dfhack_color },
     { "is_interactive", lua_dfhack_is_interactive },
-    { "lineedit", lua_dfhack_lineedit },
+    { "lineedit", dfhack_lineedit },
-    { "interpreter", lua_dfhack_interpreter },
+    { "safecall", dfhack_safecall },
-    { "safecall", lua_dfhack_safecall },
+    { "saferesume", dfhack_saferesume },
     { "onerror", dfhack_onerror },
     { "call_with_finalizer", dfhack_call_with_finalizer },
     { "with_suspend", lua_dfhack_with_suspend },
@@ -777,6 +1078,138 @@ static const luaL_Reg dfhack_funcs[] = {
     { NULL, NULL }
 };
 
+static const luaL_Reg dfhack_coro_funcs[] = {
+    { "resume", dfhack_coresume },
+    { "wrap", dfhack_cowrap },
+    { NULL, NULL }
+};
+
+/************
+ *  Events  *
+ ************/
+
+static int DFHACK_EVENT_META_TOKEN = 0;
+
+int DFHack::Lua::NewEvent(lua_State *state)
+{
+    lua_newtable(state);
+    lua_rawgetp(state, LUA_REGISTRYINDEX, &DFHACK_EVENT_META_TOKEN);
+    lua_setmetatable(state, -2);
+    return 1;
+}
+
+static void dfhack_event_invoke(lua_State *L, int base, bool from_c)
+{
+    int event = base+1;
+    int num_args = lua_gettop(L)-event;
+
+    int errorfun = base+2;
+    lua_pushcfunction(L, dfhack_onerror);
+    lua_insert(L, errorfun);
+
+    int argbase = base+3;
+    lua_pushnil(L);
+
+    // stack: |base| event errorfun (args) key cb (args)
+
+    while (lua_next(L, event))
+    {
+        if (from_c && lua_islightuserdata(L, -1) && !lua_touserdata(L, -1))
+            continue;
+
+        for (int i = 0; i < num_args; i++)
+            lua_pushvalue(L, argbase+i);
+
+        if (lua_pcall(L, num_args, 0, errorfun) != LUA_OK)
+        {
+            report_error(L);
+            lua_pop(L, 1);
+        }
+    }
+
+    lua_settop(L, base);
+}
+
+static int dfhack_event_call(lua_State *state)
+{
+    luaL_checktype(state, 1, LUA_TTABLE);
+    luaL_checkstack(state, lua_gettop(state)+2, "stack overflow in event dispatch");
+
+    dfhack_event_invoke(state, 0, false);
+    return 0;
+}
+
+void DFHack::Lua::InvokeEvent(color_ostream &out, lua_State *state, void *key, int num_args)
+{
+    AssertCoreSuspend(state);
+
+    int base = lua_gettop(state) - num_args;
+
+    if (!lua_checkstack(state, num_args+4))
+    {
+        out.printerr("Stack overflow in Lua::InvokeEvent");
+        lua_settop(state, base);
+        return;
+    }
+
+    lua_rawgetp(state, LUA_REGISTRYINDEX, key);
+
+    if (!lua_istable(state, -1))
+    {
+        if (!lua_isnil(state, -1))
+            out.printerr("Invalid event object in Lua::InvokeEvent");
+        lua_settop(state, base);
+        return;
+    }
+
+    lua_insert(state, base+1);
+
+    color_ostream *cur_out = Lua::GetOutput(state);
+    set_dfhack_output(state, &out);
+    dfhack_event_invoke(state, base, true);
+    set_dfhack_output(state, cur_out);
+}
+
+void DFHack::Lua::MakeEvent(lua_State *state, void *key)
+{
+    lua_rawgetp(state, LUA_REGISTRYINDEX, key);
+
+    if (lua_isnil(state, -1))
+    {
+        lua_pop(state, 1);
+        NewEvent(state);
+    }
+
+    lua_dup(state);
+    lua_rawsetp(state, LUA_REGISTRYINDEX, key);
+}
+
+void DFHack::Lua::Notification::invoke(color_ostream &out, int nargs)
+{
+    assert(state);
+    InvokeEvent(out, state, key, nargs);
+}
+
+void DFHack::Lua::Notification::bind(lua_State *state, void *key)
+{
+    this->state = state;
+    this->key = key;
+}
+
+void DFHack::Lua::Notification::bind(lua_State *state, const char *name)
+{
+    MakeEvent(state, this);
+
+    if (handler)
+    {
+        PushFunctionWrapper(state, 0, name, handler);
+        lua_rawsetp(state, -2, NULL);
+    }
+
+    this->state = state;
+    this->key = this;
+}
+
 /************************
  *  Main Open function  *
  ************************/
@@ -791,6 +1224,10 @@ lua_State *DFHack::Lua::Open(color_ostream &out, lua_State *state)
     luaL_openlibs(state);
     AttachDFGlobals(state);
 
+    // Table of query coroutines
+    lua_newtable(state);
+    lua_rawsetp(state, LUA_REGISTRYINDEX, &DFHACK_QUERY_COROTABLE_TOKEN);
+
     // Replace the print function of the standard library
     lua_pushcfunction(state, lua_dfhack_println);
     lua_setglobal(state, "print");
@@ -798,6 +1235,9 @@ lua_State *DFHack::Lua::Open(color_ostream &out, lua_State *state)
     // Create the dfhack global
     lua_newtable(state);
 
+    lua_pushboolean(state, IsCoreContext(state));
+    lua_setfield(state, -2, "is_core_context");
+
     // Create the metatable for exceptions
     lua_newtable(state);
     lua_pushcfunction(state, dfhack_exception_tostring);
@@ -806,6 +1246,15 @@ lua_State *DFHack::Lua::Open(color_ostream &out, lua_State *state)
     lua_rawsetp(state, LUA_REGISTRYINDEX, &DFHACK_EXCEPTION_META_TOKEN);
     lua_setfield(state, -2, "exception");
 
+    lua_newtable(state);
+    lua_pushcfunction(state, dfhack_event_call);
+    lua_setfield(state, -2, "__call");
+    lua_pushcfunction(state, Lua::NewEvent);
+    lua_setfield(state, -2, "new");
+    lua_dup(state);
+    lua_rawsetp(state, LUA_REGISTRYINDEX, &DFHACK_EVENT_META_TOKEN);
+    lua_setfield(state, -2, "event");
+
     // Initialize the dfhack global
     luaL_setfuncs(state, dfhack_funcs, 0);
 
@@ -813,9 +1262,45 @@ lua_State *DFHack::Lua::Open(color_ostream &out, lua_State *state)
 
     lua_setglobal(state, "dfhack");
 
+    // stash the loaded module table into our own registry key
+    lua_getglobal(state, "package");
+    assert(lua_istable(state, -1));
+    lua_getfield(state, -1, "loaded");
+    assert(lua_istable(state, -1));
+    lua_rawsetp(state, LUA_REGISTRYINDEX, &DFHACK_LOADED_TOKEN);
+    lua_pop(state, 1);
+
+    // replace some coroutine functions
+    lua_getglobal(state, "coroutine");
+    luaL_setfuncs(state, dfhack_coro_funcs, 0);
+    lua_pop(state, 1);
+
     // load dfhack.lua
     Require(out, state, "dfhack");
 
+    lua_settop(state, 0);
+    if (!lua_checkstack(state, 64))
+        out.printerr("Could not extend initial lua stack size to 64 items.\n");
+
     return state;
 }
 
+void DFHack::Lua::Core::Init(color_ostream &out)
+{
+    if (State)
+        return;
+
+    State = luaL_newstate();
+    Lua::Open(out, State);
+}
+
+void DFHack::Lua::Core::Reset(color_ostream &out, const char *where)
+{
+    int top = lua_gettop(State);
+
+    if (top != 0)
+    {
+        out.printerr("Common lua context stack top left at %d after %s.\n", top, where);
+        lua_settop(State, 0);
+    }
+}

library/LuaTypes.cpp

@@ -1036,7 +1036,9 @@ static int meta_call_function(lua_State *state)
 
 int LuaWrapper::method_wrapper_core(lua_State *state, function_identity_base *id)
 {
-    if (lua_gettop(state) != id->getNumArgs())
+    if (id->adjustArgs())
+        lua_settop(state, id->getNumArgs());
+    else if (lua_gettop(state) != id->getNumArgs())
         field_error(state, UPVAL_METHOD_NAME, "invalid argument count", "invoke");
 
     try {
@@ -1056,9 +1058,9 @@ int LuaWrapper::method_wrapper_core(lua_State *state, function_identity_base *id
 }
 
 /**
- * Create a closure invoking the given function, and add it to the field table.
+ * Push a closure invoking the given function.
  */
-static void AddMethodWrapper(lua_State *state, int meta_idx, int field_idx,
+void LuaWrapper::PushFunctionWrapper(lua_State *state, int meta_idx,
                                      const char *name, function_identity_base *fun)
 {
     lua_rawgetp(state, LUA_REGISTRYINDEX, &DFHACK_TYPETABLE_TOKEN);
@@ -1069,7 +1071,15 @@ static void AddMethodWrapper(lua_State *state, int meta_idx, int field_idx,
     lua_pushfstring(state, "%s()", name);
     lua_pushlightuserdata(state, fun);
     lua_pushcclosure(state, meta_call_function, 4);
+}
 
+/**
+ * Create a closure invoking the given function, and add it to the field table.
+ */
+static void AddMethodWrapper(lua_State *state, int meta_idx, int field_idx,
+                             const char *name, function_identity_base *fun)
+{
+    PushFunctionWrapper(state, meta_idx, name, fun);
     lua_setfield(state, field_idx, name);
 }
 

library/PluginManager.cpp

@@ -33,6 +33,7 @@ distribution.
 #include "MiscUtils.h"
 
 #include "LuaWrapper.h"
+#include "LuaTools.h"
 
 using namespace DFHack;
 
@@ -380,6 +381,7 @@ command_result Plugin::on_update(color_ostream &out)
     if(state == PS_LOADED && plugin_onupdate)
     {
         cr = plugin_onupdate(out);
+        Lua::Core::Reset(out, "plugin_onupdate");
     }
     access->lock_sub();
     return cr;
@@ -392,6 +394,7 @@ command_result Plugin::on_state_change(color_ostream &out, state_change_event ev
     if(state == PS_LOADED && plugin_onstatechange)
     {
         cr = plugin_onstatechange(out, event);
+        Lua::Core::Reset(out, "plugin_onstatechange");
     }
     access->lock_sub();
     return cr;
@@ -445,9 +448,7 @@ void Plugin::index_lua(DFLibrary *lib)
         for (; cmdlist->name; ++cmdlist)
         {
             auto &cmd = lua_commands[cmdlist->name];
-            if (!cmd) cmd = new LuaCommand;
+            if (!cmd) cmd = new LuaCommand(this,cmdlist->name);
-            cmd->owner = this;
-            cmd->name = cmdlist->name;
             cmd->command = cmdlist->command;
         }
     }
@@ -456,12 +457,22 @@ void Plugin::index_lua(DFLibrary *lib)
         for (; funlist->name; ++funlist)
         {
             auto &cmd = lua_functions[funlist->name];
-            if (!cmd) cmd = new LuaFunction;
+            if (!cmd) cmd = new LuaFunction(this,funlist->name);
-            cmd->owner = this;
-            cmd->name = funlist->name;
             cmd->identity = funlist->identity;
         }
     }
+    if (auto evlist = (EventReg*)LookupPlugin(lib, "plugin_lua_events"))
+    {
+        for (; evlist->name; ++evlist)
+        {
+            auto &cmd = lua_events[evlist->name];
+            if (!cmd) cmd = new LuaEvent(this,evlist->name);
+            cmd->handler.identity = evlist->event->get_handler();
+            cmd->event = evlist->event;
+            if (cmd->active)
+                cmd->event->bind(Lua::Core::State, cmd);
+        }
+    }
 }
 
 void Plugin::reset_lua()
@@ -470,6 +481,11 @@ void Plugin::reset_lua()
         it->second->command = NULL;
     for (auto it = lua_functions.begin(); it != lua_functions.end(); ++it)
         it->second->identity = NULL;
+    for (auto it = lua_events.begin(); it != lua_events.end(); ++it)
+    {
+        it->second->handler.identity = NULL;
+        it->second->event = NULL;
+    }
 }
 
 int Plugin::lua_cmd_wrapper(lua_State *state)
@@ -512,14 +528,36 @@ void Plugin::open_lua(lua_State *state, int table)
     }
 
     for (auto it = lua_functions.begin(); it != lua_functions.end(); ++it)
+    {
+        push_function(state, it->second);
+        lua_setfield(state, table, it->first.c_str());
+    }
+
+    if (Lua::IsCoreContext(state))
+    {
+        for (auto it = lua_events.begin(); it != lua_events.end(); ++it)
+        {
+            Lua::MakeEvent(state, it->second);
+
+            push_function(state, &it->second->handler);
+            lua_rawsetp(state, -2, NULL);
+
+            it->second->active = true;
+            if (it->second->event)
+                it->second->event->bind(state, it->second);
+
+            lua_setfield(state, table, it->first.c_str());
+        }
+    }
+}
+
+void Plugin::push_function(lua_State *state, LuaFunction *fn)
 {
     lua_rawgetp(state, LUA_REGISTRYINDEX, &LuaWrapper::DFHACK_TYPETABLE_TOKEN);
     lua_pushlightuserdata(state, NULL);
-        lua_pushfstring(state, "%s.%s()", name.c_str(), it->second->name.c_str());
+    lua_pushfstring(state, "%s.%s()", name.c_str(), fn->name.c_str());
-        lua_pushlightuserdata(state, it->second);
+    lua_pushlightuserdata(state, fn);
     lua_pushcclosure(state, lua_fun_wrapper, 4);
-        lua_setfield(state, table, it->first.c_str());
-    }
 }
 
 PluginManager::PluginManager(Core * core)

library/include/Core.h

@@ -90,6 +90,8 @@ namespace DFHack
             static Core instance;
             return instance;
         }
+        /// check if the activity lock is owned by this thread
+        bool isSuspended(void);
         /// try to acquire the activity lock
         void Suspend(void);
         /// return activity lock

library/include/DataFuncs.h

@@ -161,15 +161,15 @@ INSTANTIATE_WRAPPERS(5, (A1,A2,A3,A4,A5), (vA1,vA2,vA3,vA4,vA5),
     public:
         typedef function_wrapper<T> wrapper;
 
-        function_identity(T ptr)
+        function_identity(T ptr, bool vararg)
-            : function_identity_base(wrapper::num_args), ptr(ptr) {};
+            : function_identity_base(wrapper::num_args, vararg), ptr(ptr) {};
 
         virtual void invoke(lua_State *state, int base) { wrapper::execute(state, base, ptr); }
     };
 
     template<class T>
-    inline function_identity_base *wrap_function(T ptr) {
+    inline function_identity_base *wrap_function(T ptr, bool vararg = false) {
         // bah, but didn't have any idea how to allocate statically
-        return new function_identity<T>(ptr);
+        return new function_identity<T>(ptr, vararg);
     }
 }

library/include/DataIdentity.h

@@ -39,13 +39,17 @@ namespace DFHack
 {
     class DFHACK_EXPORT function_identity_base : public type_identity {
         int num_args;
+        bool vararg;
 
     public:
-        function_identity_base(int num_args) : type_identity(0), num_args(num_args) {};
+        function_identity_base(int num_args, bool vararg = false)
+            : type_identity(0), num_args(num_args), vararg(vararg) {};
 
         virtual identity_type type() { return IDTYPE_FUNCTION; }
 
         int getNumArgs() { return num_args; }
+        bool adjustArgs() { return vararg; }
+
         std::string getFullName() { return "function"; }
 
         virtual void invoke(lua_State *state, int base) = 0;

library/include/LuaTools.h

@@ -34,6 +34,10 @@ distribution.
 #include <lua.h>
 #include <lauxlib.h>
 
+namespace DFHack {
+    class function_identity_base;
+}
+
 namespace DFHack {namespace Lua {
     /**
      * Create or initialize a lua interpreter with access to DFHack tools.
@@ -41,11 +45,22 @@ namespace DFHack { namespace Lua {
     DFHACK_EXPORT lua_State *Open(color_ostream &out, lua_State *state = NULL);
 
     /**
-     * Load a module using require().
+     * Load a module using require(). Leaves the stack as is.
      */
     DFHACK_EXPORT bool Require(color_ostream &out, lua_State *state,
                                const std::string &module, bool setglobal = false);
 
+    /**
+     * Push the module table, loading it using require() if necessary.
+     */
+    DFHACK_EXPORT bool PushModule(color_ostream &out, lua_State *state, const char *module);
+
+    /**
+     * Push the public object name exported by the module. Uses PushModule.
+     */
+    DFHACK_EXPORT bool PushModulePublic(color_ostream &out, lua_State *state,
+                                        const char *module, const char *name);
+
     /**
      * Check if the object at the given index is NIL or NULL.
      */
@@ -79,6 +94,12 @@ namespace DFHack { namespace Lua {
      */
     DFHACK_EXPORT void *GetDFObject(lua_State *state, type_identity *type, int val_index, bool exact_type = false);
 
+    /**
+     * Check that the value is a wrapped DF object of the given type, and if so return the pointer.
+     * Otherwise throw an argument type error.
+     */
+    DFHACK_EXPORT void *CheckDFObject(lua_State *state, type_identity *type, int val_index, bool exact_type = false);
+
     /**
      * Assign the value at val_index to the target of given identity using df.assign().
      * Return behavior is of SafeCall below.
@@ -102,6 +123,14 @@ namespace DFHack { namespace Lua {
         return (T*)GetDFObject(state, df::identity_traits<T>::get(), val_index, exact_type);
     }
 
+    /**
+     * Check that the value is a wrapped DF object of the correct type, and if so return the pointer. Otherwise throw an argument type error.
+     */
+    template<class T>
+    T *CheckDFObject(lua_State *state, int val_index, bool exact_type = false) {
+        return (T*)CheckDFObject(state, df::identity_traits<T>::get(), val_index, exact_type);
+    }
+
     /**
      * Assign the value at val_index to the target using df.assign().
      */
@@ -110,12 +139,56 @@ namespace DFHack { namespace Lua {
         return AssignDFObject(out, state, df::identity_traits<T>::get(), target, val_index, perr);
     }
 
+    /**
+     * Check if the status is a success, i.e. LUA_OK or LUA_YIELD.
+     */
+    inline bool IsSuccess(int status) {
+        return (status == LUA_OK || status == LUA_YIELD);
+    }
+
+    // Internal helper
+    template<int (*cb)(lua_State*,int,int)>
+    int TailPCallK_Thunk(lua_State *state) {
+        int tmp;
+        int rv = lua_getctx(state, &tmp);
+        return cb(state, rv, tmp);
+    }
+
+    /**
+     * A utility for using the restartable pcall feature more conveniently;
+     * specifically, the callback is called with the same kind of arguments
+     * in both yield and non-yield case.
+     */
+    template<int (*cb)(lua_State*,int,int)>
+    int TailPCallK(lua_State *state, int narg, int nret, int errfun, int ctx) {
+        int rv = lua_pcallk(state, narg, nret, errfun, ctx, &TailPCallK_Thunk<cb>);
+        return cb(state, rv, ctx);
+    }
+
     /**
      * Invoke lua function via pcall. Returns true if success.
      * If an error is signalled, and perr is true, it is printed and popped from the stack.
      */
     DFHACK_EXPORT bool SafeCall(color_ostream &out, lua_State *state, int nargs, int nres, bool perr = true);
 
+    /**
+     * Pops a function from the top of the stack, and pushes a new coroutine.
+     */
+    DFHACK_EXPORT lua_State *NewCoroutine(lua_State *state);
+
+    /**
+     * Resume the coroutine using nargs values from state from. Results or the error are moved back.
+     * If an error is signalled, and perr is true, it is printed and popped from the stack.
+     * Returns the lua_resume return value.
+     */
+    DFHACK_EXPORT int SafeResume(color_ostream &out, lua_State *from, lua_State *thread, int nargs, int nres, bool perr = true);
+
+    /**
+     * Works just like SafeCall, only expects a coroutine on the stack
+     * instead of a function. Returns the lua_resume return value.
+     */
+    DFHACK_EXPORT int SafeResume(color_ostream &out, lua_State *from, int nargs, int nres, bool perr = true);
+
     /**
      * Parse code from string with debug_tag and env_idx, then call it using SafeCall.
      * In case of error, it is either left on the stack, or printed like SafeCall does.
@@ -131,8 +204,165 @@ namespace DFHack { namespace Lua {
 
     /**
      * Run an interactive interpreter loop if possible, or return false.
+     * Uses RunCoreQueryLoop internally.
      */
     DFHACK_EXPORT bool InterpreterLoop(color_ostream &out, lua_State *state,
-                                       const char *prompt = NULL, int env = 0, const char *hfile = NULL);
+                                       const char *prompt = NULL, const char *hfile = NULL);
+
+    /**
+     * Run an interactive prompt loop. All access to the lua state
+     * is done inside CoreSuspender, while waiting for input happens
+     * without the suspend lock.
+     */
+    DFHACK_EXPORT bool RunCoreQueryLoop(color_ostream &out, lua_State *state,
+                                        bool (*init)(color_ostream&, lua_State*, void*),
+                                        void *arg);
+
+    /**
+     * Push utility functions
+     */
+#if 0
+#define NUMBER_PUSH(type) inline void Push(lua_State *state, type value) { lua_pushnumber(state, value); }
+    NUMBER_PUSH(char)
+    NUMBER_PUSH(int8_t) NUMBER_PUSH(uint8_t)
+    NUMBER_PUSH(int16_t) NUMBER_PUSH(uint16_t)
+    NUMBER_PUSH(int32_t) NUMBER_PUSH(uint32_t)
+    NUMBER_PUSH(int64_t) NUMBER_PUSH(uint64_t)
+    NUMBER_PUSH(float) NUMBER_PUSH(double)
+#undef NUMBER_PUSH
+#else
+    template<class T> inline void Push(lua_State *state, T value) {
+        lua_pushnumber(state, lua_Number(value));
+    }
+#endif
+    inline void Push(lua_State *state, bool value) {
+        lua_pushboolean(state, value);
+    }
+    inline void Push(lua_State *state, const std::string &str) {
+        lua_pushlstring(state, str.data(), str.size());
+    }
+    inline void Push(lua_State *state, df::coord &obj) { PushDFObject(state, &obj); }
+    inline void Push(lua_State *state, df::coord2d &obj) { PushDFObject(state, &obj); }
+    template<class T> inline void Push(lua_State *state, T *ptr) {
+        PushDFObject(state, ptr);
+    }
+
+    template<class T>
+    void PushVector(lua_State *state, const T &pvec)
+    {
+        lua_createtable(state,pvec.size(),0);
+
+        for (size_t i = 0; i < pvec.size(); i++)
+        {
+            Push(state, pvec[i]);
+            lua_rawseti(state, -2, i+1);
+        }
+    }
+
+    DFHACK_EXPORT int PushPosXYZ(lua_State *state, df::coord pos);
+
+    DFHACK_EXPORT bool IsCoreContext(lua_State *state);
+
+    DFHACK_EXPORT int NewEvent(lua_State *state);
+    DFHACK_EXPORT void MakeEvent(lua_State *state, void *key);
+    DFHACK_EXPORT void InvokeEvent(color_ostream &out, lua_State *state, void *key, int num_args);
+
+    /**
+     * Namespace for the common lua interpreter state.
+     * All accesses must be done under CoreSuspender.
+     */
+    namespace Core {
+        DFHACK_EXPORT extern lua_State *State;
+
+        // Not exported; for use by the Core class
+        void Init(color_ostream &out);
+        void Reset(color_ostream &out, const char *where);
+
+        template<class T> inline void Push(T &arg) { Lua::Push(State, arg); }
+        template<class T> inline void Push(const T &arg) { Lua::Push(State, arg); }
+        template<class T> inline void PushVector(const T &arg) { Lua::PushVector(State, arg); }
+
+        inline bool SafeCall(color_ostream &out, int nargs, int nres, bool perr = true) {
+            return Lua::SafeCall(out, State, nargs, nres, perr);
+        }
+        inline bool PushModule(color_ostream &out, const char *module) {
+            return Lua::PushModule(out, State, module);
+        }
+        inline bool PushModulePublic(color_ostream &out, const char *module, const char *name) {
+            return Lua::PushModulePublic(out, State, module, name);
+        }
+    }
+
+    class DFHACK_EXPORT Notification {
+        lua_State *state;
+        void *key;
+        function_identity_base *handler;
+
+    public:
+        Notification(function_identity_base *handler = NULL)
+            : state(NULL), key(NULL), handler(handler) {}
+
+        lua_State *get_state() { return state; }
+        function_identity_base *get_handler() { return handler; }
+
+        void invoke(color_ostream &out, int nargs);
+
+        void bind(lua_State *state, const char *name);
+        void bind(lua_State *state, void *key);
+    };
 }}
 
+#define DEFINE_LUA_EVENT_0(name, handler) \
+    static DFHack::Lua::Notification name##_event(df::wrap_function(handler, true)); \
+    void name(color_ostream &out) { \
+        handler(out); \
+        if (name##_event.get_state()) { \
+            name##_event.invoke(out, 0); \
+        } \
+    }
+
+#define DEFINE_LUA_EVENT_1(name, handler, arg_type1) \
+    static DFHack::Lua::Notification name##_event(df::wrap_function(handler, true)); \
+    void name(color_ostream &out, arg_type1 arg1) { \
+        handler(out, arg1); \
+        if (auto state = name##_event.get_state()) { \
+            DFHack::Lua::Push(state, arg1); \
+            name##_event.invoke(out, 1); \
+        } \
+    }
+
+#define DEFINE_LUA_EVENT_2(name, handler, arg_type1, arg_type2) \
+    static DFHack::Lua::Notification name##_event(df::wrap_function(handler, true)); \
+    void name(color_ostream &out, arg_type1 arg1, arg_type2 arg2) { \
+        handler(out, arg1, arg2); \
+        if (auto state = name##_event.get_state()) { \
+            DFHack::Lua::Push(state, arg1); \
+            DFHack::Lua::Push(state, arg2); \
+            name##_event.invoke(out, 2); \
+        } \
+    }
+
+#define DEFINE_LUA_EVENT_3(name, handler, arg_type1, arg_type2, arg_type3) \
+    static DFHack::Lua::Notification name##_event(df::wrap_function(handler, true)); \
+    void name(color_ostream &out, arg_type1 arg1, arg_type2 arg2, arg_type3 arg3) { \
+        handler(out, arg1, arg2, arg3); \
+        if (auto state = name##_event.get_state()) { \
+            DFHack::Lua::Push(state, arg1); \
+            DFHack::Lua::Push(state, arg2); \
+            DFHack::Lua::Push(state, arg3); \
+            name##_event.invoke(out, 3); \
+        } \
+    }
+
+#define DEFINE_LUA_EVENT_4(name, handler, arg_type1, arg_type2, arg_type3, arg_type4) \
+    static DFHack::Lua::Notification name##_event(df::wrap_function(handler, true)); \
+    void name(color_ostream &out, arg_type1 arg1, arg_type2 arg2, arg_type3 arg3, arg_type4 arg4) { \
+        handler(out, arg1, arg2, arg3, arg4); \
+        if (auto state = name##_event.get_state()) { \
+            DFHack::Lua::Push(state, arg1); \
+            DFHack::Lua::Push(state, arg2); \
+            DFHack::Lua::Push(state, arg3); \
+            DFHack::Lua::Push(state, arg4); \
+            name##_event.invoke(out, 4); \
+        } \
+    }

library/include/LuaWrapper.h

@@ -222,6 +222,12 @@ namespace DFHack { namespace LuaWrapper {
      */
     void AttachEnumKeys(lua_State *state, int meta_idx, int ftable_idx, type_identity *ienum);
 
+    /**
+     * Push a closure invoking the given function.
+     */
+    void PushFunctionWrapper(lua_State *state, int meta_idx,
+                             const char *name, function_identity_base *fun);
+
     /**
      * Wrap functions and add them to the table on the top of the stack.
      */

library/include/PluginManager.h

@@ -51,6 +51,9 @@ namespace DFHack
     class virtual_identity;
     class RPCService;
     class function_identity_base;
+    namespace Lua {
+        class Notification;
+    }
 
     // anon type, pretty much
     struct DFLibrary;
@@ -80,6 +83,10 @@ namespace DFHack
         const char *name;
         function_identity_base *identity;
     };
+    struct DFHACK_EXPORT EventReg {
+        const char *name;
+        Lua::Notification *event;
+    };
     struct DFHACK_EXPORT PluginCommand
     {
         typedef command_result (*command_function)(color_ostream &out, std::vector <std::string> &);
@@ -178,6 +185,7 @@ namespace DFHack
             Plugin *owner;
             std::string name;
             int (*command)(lua_State *state);
+            LuaCommand(Plugin *owner, std::string name) : owner(owner), name(name) {}
         };
         std::map<std::string, LuaCommand*> lua_commands;
         static int lua_cmd_wrapper(lua_State *state);
@@ -186,9 +194,19 @@ namespace DFHack
             Plugin *owner;
             std::string name;
             function_identity_base *identity;
+            LuaFunction(Plugin *owner, std::string name) : owner(owner), name(name) {}
         };
         std::map<std::string, LuaFunction*> lua_functions;
         static int lua_fun_wrapper(lua_State *state);
+        void push_function(lua_State *state, LuaFunction *fn);
+
+        struct LuaEvent {
+            LuaFunction handler;
+            Lua::Notification *event;
+            bool active;
+            LuaEvent(Plugin *owner, std::string name) : handler(owner,name), active(false) {}
+        };
+        std::map<std::string, LuaEvent*> lua_events;
 
         void index_lua(DFLibrary *lib);
         void reset_lua();
@@ -253,7 +271,10 @@ namespace DFHack
     DFhackCExport const DFHack::CommandReg plugin_lua_commands[] =
 #define DFHACK_PLUGIN_LUA_FUNCTIONS \
     DFhackCExport const DFHack::FunctionReg plugin_lua_functions[] =
+#define DFHACK_PLUGIN_LUA_EVENTS \
+    DFhackCExport const DFHack::EventReg plugin_lua_events[] =
 
 #define DFHACK_LUA_COMMAND(name) { #name, name }
-#define DFHACK_LUA_FUNCTION(name) { #name, df::wrap_function(name) }
+#define DFHACK_LUA_FUNCTION(name) { #name, df::wrap_function(name,true) }
+#define DFHACK_LUA_EVENT(name) { #name, &name##_event }
 #define DFHACK_LUA_END { NULL, NULL }

library/include/RemoteClient.h

@@ -66,7 +66,7 @@ namespace  DFHack
     };
 
     struct RPCMessageHeader {
-        static const int MAX_MESSAGE_SIZE = 8*1048756;
+        static const int MAX_MESSAGE_SIZE = 8*1048576;
 
         int16_t id;
         int32_t size;

library/lua/dfhack.lua

@@ -105,6 +105,10 @@ function xyz2pos(x,y,z)
     end
 end
 
+function dfhack.event:__tostring()
+    return "<event>"
+end
+
 function dfhack.persistent:__tostring()
     return "<persistent "..self.entry_id..":"..self.key.."=\""
            ..self.value.."\":"..table.concat(self.ints,",")..">"
@@ -124,5 +128,83 @@ function dfhack.maps.getTileSize()
     return map.x_count, map.y_count, map.z_count
 end
 
+-- Interactive
+
+local print_banner = true
+
+function dfhack.interpreter(prompt,hfile,env)
+    if not dfhack.is_interactive() then
+        return nil, 'not interactive'
+    end
+
+    print("Type quit to exit interactive lua interpreter.")
+
+    if print_banner then
+        print("Shortcuts:\n"..
+              " '= foo' => '_1,_2,... = foo'\n"..
+              " '! foo' => 'print(foo)'\n"..
+              "Both save the first result as '_'.")
+        print_banner = false
+    end
+
+    local prompt_str = "["..(prompt or 'lua').."]# "
+    local prompt_env = {}
+	local t_prompt
+    local vcnt = 1
+
+    setmetatable(prompt_env, { __index = env or _G })
+	local cmdlinelist={}
+    while true do
+        local cmdline = dfhack.lineedit(t_prompt or prompt_str, hfile)
+
+        if cmdline == nil or cmdline == 'quit' then
+            break
+        elseif cmdline ~= '' then
+            local pfix = string.sub(cmdline,1,1)
+
+            if pfix == '!' or pfix == '=' then
+                cmdline = 'return '..string.sub(cmdline,2)
+            end
+			table.insert(cmdlinelist,cmdline)
+            local code,err = load(table.concat(cmdlinelist,'\n'), '=(interactive)', 't', prompt_env)
+
+            if code == nil then
+				if err:sub(-5)=="<eof>" then
+					t_prompt="[cont]"
+					
+				else
+					dfhack.printerr(err)
+					cmdlinelist={}
+					t_prompt=nil
+				end
+            else
+			
+				cmdlinelist={}
+				t_prompt=nil
+				
+                local data = table.pack(safecall(code))
+
+                if data[1] and data.n > 1 then
+                    prompt_env._ = data[2]
+
+                    if pfix == '!' then
+                        safecall(print, table.unpack(data,2,data.n))
+                    else
+                        for i=2,data.n do
+                            local varname = '_'..vcnt
+                            prompt_env[varname] = data[i]
+                            dfhack.print(varname..' = ')
+                            safecall(print, data[i])
+                            vcnt = vcnt + 1
+                        end
+                    end
+                end
+            end
+        end
+    end
+
+    return true
+end
+
 -- Feed the table back to the require() mechanism.
 return dfhack

plugins/Dfusion/dfusion.cpp

@@ -121,6 +121,12 @@ command_result lua_run_file (color_ostream &out, std::vector <std::string> &para
 }
 command_result lua_run (color_ostream &out, std::vector <std::string> &parameters)
 {
+    if (!parameters.empty() && parameters[0] == "--core-context")
+    {
+        Lua::InterpreterLoop(out, Lua::Core::State, "core lua");
+        return CR_OK;
+    }
+
 	mymutex->lock();
 	lua::state s=lua::glua::Get();
 

plugins/burrows.cpp

@@ -5,6 +5,7 @@
 #include "Error.h"
 
 #include "DataFuncs.h"
+#include "LuaTools.h"
 
 #include "modules/Gui.h"
 #include "modules/Job.h"
@@ -121,6 +122,8 @@ static int name_burrow_id = -1;
 
 static void handle_burrow_rename(color_ostream &out, df::burrow *burrow);
 
+DEFINE_LUA_EVENT_1(onBurrowRename, handle_burrow_rename, df::burrow*);
+
 static void detect_burrow_renames(color_ostream &out)
 {
     if (ui->main.mode == ui_sidebar_mode::Burrows &&
@@ -134,7 +137,7 @@ static void detect_burrow_renames(color_ostream &out)
         auto burrow = df::burrow::find(name_burrow_id);
         name_burrow_id = -1;
         if (burrow)
-            handle_burrow_rename(out, burrow);
+            onBurrowRename(out, burrow);
     }
 }
 
@@ -151,6 +154,9 @@ static std::map<int,DigJob> diggers;
 static void handle_dig_complete(color_ostream &out, df::job_type job, df::coord pos,
                                 df::tiletype old_tile, df::tiletype new_tile);
 
+DEFINE_LUA_EVENT_4(onDigComplete, handle_dig_complete,
+                   df::job_type, df::coord, df::tiletype, df::tiletype);
+
 static void detect_digging(color_ostream &out)
 {
     for (auto it = diggers.begin(); it != diggers.end();)
@@ -172,7 +178,7 @@ static void detect_digging(color_ostream &out)
 
                 if (new_tile != it->second.old_tile)
                 {
-                    handle_dig_complete(out, it->second.job, pos, it->second.old_tile, new_tile);
+                    onDigComplete(out, it->second.job, pos, it->second.old_tile, new_tile);
 
                     //if (worker && !worker->job.current_job)
                     //    worker->counters.think_counter = worker->counters.job_counter = 0;
@@ -410,6 +416,17 @@ static void handle_dig_complete(color_ostream &out, df::job_type job, df::coord
     }
 }
 
+static void renameBurrow(color_ostream &out, df::burrow *burrow, std::string name)
+{
+    CHECK_NULL_POINTER(burrow);
+
+    // The event makes this absolutely necessary
+    CoreSuspender suspend;
+
+    burrow->name = name;
+    onBurrowRename(out, burrow);
+}
+
 static df::burrow *findByName(color_ostream &out, std::string name, bool silent = false)
 {
     int id = -1;
@@ -552,6 +569,7 @@ static bool setTilesByKeyword(df::burrow *target, std::string name, bool enable)
 }
 
 DFHACK_PLUGIN_LUA_FUNCTIONS {
+    DFHACK_LUA_FUNCTION(renameBurrow),
     DFHACK_LUA_FUNCTION(findByName),
     DFHACK_LUA_FUNCTION(copyUnits),
     DFHACK_LUA_FUNCTION(copyTiles),
@@ -559,6 +577,12 @@ DFHACK_PLUGIN_LUA_FUNCTIONS {
     DFHACK_LUA_END
 };
 
+DFHACK_PLUGIN_LUA_EVENTS {
+    DFHACK_LUA_EVENT(onBurrowRename),
+    DFHACK_LUA_EVENT(onDigComplete),
+    DFHACK_LUA_END
+};
+
 static command_result burrow(color_ostream &out, vector <string> &parameters)
 {
     CoreSuspender suspend;