Improve support for void RPC functions, dfhack-run, etc.

CMakeLists.txt

@@ -53,7 +53,7 @@ set(DF_VERSION_MINOR "34")
 set(DF_VERSION_PATCH "05")
 set(DF_VERSION "${DF_VERSION_MAJOR}.${DF_VERSION_MINOR}.${DF_VERSION_PATCH}")
 
-set(DFHACK_RELEASE "1d")
+set(DFHACK_RELEASE "1e")
 
 set(DFHACK_VERSION "${DF_VERSION_MAJOR}.${DF_VERSION_MINOR}.${DF_VERSION_PATCH}-r${DFHACK_RELEASE}")
 add_definitions(-DDFHACK_VERSION="${DFHACK_VERSION}")

library/CMakeLists.txt

@@ -159,8 +159,8 @@ ENDIF()
 # Protobuf
 FILE(GLOB PROJECT_PROTOS ${CMAKE_CURRENT_SOURCE_DIR}/proto/*.proto)
 
-STRING(REPLACE ".proto" ".pb.cc" PROJECT_PROTO_SRCS ${PROJECT_PROTOS})
+STRING(REPLACE ".proto" ".pb.cc" PROJECT_PROTO_SRCS "${PROJECT_PROTOS}")
-STRING(REPLACE ".proto" ".pb.h" PROJECT_PROTO_HDRS ${PROJECT_PROTOS})
+STRING(REPLACE ".proto" ".pb.h" PROJECT_PROTO_HDRS "${PROJECT_PROTOS}")
 
 LIST(APPEND PROJECT_HEADERS ${PROJECT_PROTO_HDRS})
 LIST(APPEND PROJECT_SOURCES ${PROJECT_PROTO_SRCS})
@@ -249,6 +249,8 @@ IF(UNIX)
     # On linux, copy our version of the df launch script which sets LD_PRELOAD
     install(PROGRAMS ${dfhack_SOURCE_DIR}/package/linux/dfhack
             DESTINATION .)
+    install(PROGRAMS ${dfhack_SOURCE_DIR}/package/linux/dfhack-run
+            DESTINATION .)
 ELSE()
     if(NOT BUILD_EGGY)
         # On windows, copy the renamed SDL so DF can still run.

library/ColorText.cpp

@@ -56,6 +56,8 @@ using namespace DFHack;
 #include "tinythread.h"
 using namespace tthread;
 
+bool color_ostream::log_errors_to_stderr = false;
+
 void color_ostream::flush_buffer(bool flush)
 {
     auto buffer = buf();
@@ -122,6 +124,7 @@ void color_ostream::vprinterr(const char *format, va_list args)
 {
     color_value save = cur_color;
 
+    if (log_errors_to_stderr)
         fprintf(stderr, format, args);
 
     color(COLOR_LIGHTRED);

library/Core.cpp

@@ -572,6 +572,8 @@ Core::Core()
     top_viewscreen = NULL;
     screen_window = NULL;
     server = NULL;
+
+    color_ostream::log_errors_to_stderr = true;
 };
 
 void Core::fatal (std::string output, bool deactivate)

library/RemoteClient.cpp

@@ -176,6 +176,10 @@ bool RemoteClient::connect(int port)
     bind_call.p_client = this;
     bind_call.id = 0;
 
+    runcmd_call.name = "RunCommand";
+    runcmd_call.p_client = this;
+    runcmd_call.id = 1;
+
     return true;
 }
 
@@ -220,6 +224,24 @@ bool RemoteClient::bind(color_ostream &out, RemoteFunctionBase *function,
     return true;
 }
 
+command_result RemoteClient::run_command(color_ostream &out, const std::string &cmd,
+                                         const std::vector<std::string> &args)
+{
+    if (!active || !socket.IsSocketValid())
+    {
+        out.printerr("In RunCommand: client connection not valid.\n");
+        return CR_FAILURE;
+    }
+
+    runcmd_call.reset();
+
+    runcmd_call.in()->set_command(cmd);
+    for (size_t i = 0; i < args.size(); i++)
+        runcmd_call.in()->add_arguments(args[i]);
+
+    return runcmd_call.execute(out);
+}
+
 void RPCFunctionBase::reset(bool free)
 {
     if (free)
@@ -256,11 +278,14 @@ bool RemoteFunctionBase::bind(color_ostream &out, RemoteClient *client,
     return client->bind(out, this, name, proto);
 }
 
-bool DFHack::sendRemoteMessage(CSimpleSocket &socket, int16_t id, const MessageLite *msg)
+bool DFHack::sendRemoteMessage(CSimpleSocket &socket, int16_t id, const MessageLite *msg, int *psz)
 {
     int size = msg->ByteSize();
     int fullsz = size + sizeof(RPCMessageHeader);
 
+    if (psz)
+        *psz = size;
+
     std::auto_ptr<uint8_t> data(new uint8_t[fullsz]);
     RPCMessageHeader *hdr = (RPCMessageHeader*)data.get();
 

library/RemoteServer.cpp

@@ -64,10 +64,11 @@ using dfproto::CoreTextFragment;
 using google::protobuf::MessageLite;
 
 CoreService::CoreService() {
-    // This must be the first method, so that it gets id 0
+    // These 2 methods must be first, so that they get id 0 and 1
     addMethod("BindMethod", &CoreService::BindMethod);
-
     addMethod("RunCommand", &CoreService::RunCommand);
+
+    // Add others here:
 }
 
 command_result CoreService::BindMethod(color_ostream &stream,
@@ -96,8 +97,7 @@ command_result CoreService::BindMethod(color_ostream &stream,
 }
 
 command_result CoreService::RunCommand(color_ostream &stream,
-                                       const dfproto::CoreRunCommandRequest *in,
+                                       const dfproto::CoreRunCommandRequest *in)
-                                       CoreVoidReply*)
 {
     std::string cmd = in->command();
     std::vector<std::string> args;
@@ -242,6 +242,7 @@ void ServerConnection::threadFn(void *arg)
     std::cerr << "Client connection established." << endl;
 
     while (!me->in_error) {
+        // Read the message
         RPCMessageHeader header;
 
         if (!readFullBuffer(*me->socket, &header, sizeof(header)))
@@ -269,6 +270,9 @@ void ServerConnection::threadFn(void *arg)
 
         //out.print("Handling %d:%d\n", header.id, header.size);
 
+        // Find and call the function
+        int in_size = header.size;
+
         ServerFunctionBase *fn = vector_get(me->functions, header.id);
         MessageLite *reply = NULL;
         command_result res = CR_FAILURE;
@@ -290,6 +294,7 @@ void ServerConnection::threadFn(void *arg)
             }
         }
 
+        // Flush all text output
         if (me->in_error)
             break;
 
@@ -297,9 +302,12 @@ void ServerConnection::threadFn(void *arg)
 
         //out.print("Answer %d:%d\n", res, reply);
 
+        // Send reply
+        int out_size = 0;
+
         if (res == CR_OK && reply)
         {
-            if (!sendRemoteMessage(*me->socket, RPC_REPLY_RESULT, reply))
+            if (!sendRemoteMessage(*me->socket, RPC_REPLY_RESULT, reply, &out_size))
             {
                 out.printerr("In RPC server: I/O error in send result.\n");
                 break;
@@ -307,6 +315,9 @@ void ServerConnection::threadFn(void *arg)
         }
         else
         {
+            if (reply)
+                out_size = reply->ByteSize();
+
             header.id = RPC_REPLY_FAIL;
             header.size = res;
 
@@ -316,6 +327,12 @@ void ServerConnection::threadFn(void *arg)
                 break;
             }
         }
+
+        // Cleanup
+        if (fn)
+        {
+            fn->reset(out_size > 32768 || in_size > 32768);
+        }
     }
 
     std::cerr << "Shutting down client connection." << endl;

library/dfhack-run.cpp

@@ -71,19 +71,19 @@ int main (int argc, char *argv[])
     if (!client.connect())
         return 2;
 
-    // Bind to RunCommand
+    // Call the command
-    RemoteFunction<CoreRunCommandRequest,CoreVoidReply> command;
+    std::vector<std::string> args;
+    for (int i = 2; i < argc; i++)
+        args.push_back(argv[i]);
 
-    if (!command.bind(out, &client, "RunCommand"))
+    command_result rv = client.run_command(out, argv[1], args);
-        return 2;
 
-    // Execute it
+    if (rv != CR_OK) {
-    command.in()->set_command(argv[1]);
+        if (rv == CR_NOT_IMPLEMENTED)
-    for (int i = 2; i < argc; i++)
+            out.printerr("%s is not a recognized command.\n", argv[1]);
-        command.in()->add_arguments(argv[i]);
 
-    if (command.execute(out) != CR_OK)
         return 1;
+    }
 
     out.flush();
     return 0;

library/include/ColorText.h

@@ -118,6 +118,8 @@ namespace  DFHack
 
         virtual bool is_console() { return false; }
         virtual color_ostream *proxy_target() { return NULL; }
+
+        static bool log_errors_to_stderr;
     };
 
     inline color_ostream &operator << (color_ostream &out, color_ostream::color_value clr)

library/include/RemoteClient.h

@@ -33,7 +33,7 @@ distribution.
 
 namespace  DFHack
 {
-    using dfproto::CoreVoidReply;
+    using dfproto::EmptyMessage;
 
     enum command_result
     {
@@ -64,6 +64,43 @@ namespace  DFHack
         int32_t size;
     };
 
+    /* Protocol description:
+     *
+     * 1. Handshake
+     *
+     *   Client initiates connection by sending the handshake
+     *   request header. The server responds with the response
+     *   magic. Currently both versions must be 1.
+     *
+     * 2. Interaction
+     *
+     *   Requests are done by exchanging messages between the
+     *   client and the server. Messages consist of a serialized
+     *   protobuf message preceeded by RPCMessageHeader. The size
+     *   field specifies the length of the protobuf part.
+     *
+     *   NOTE: As a special exception, RPC_REPLY_FAIL uses the size
+     *         field to hold the error code directly.
+     *
+     *   Every callable function is assigned a non-negative id by
+     *   the server. Id 0 is reserved for BindMethod, which can be
+     *   used to request any other id by function name. Id 1 is
+     *   RunCommand, used to call console commands remotely.
+     *
+     *   The client initiates every call by sending a message with
+     *   appropriate function id and input arguments. The server
+     *   responds with zero or more RPC_REPLY_TEXT:CoreTextNotification
+     *   messages, followed by RPC_REPLY_RESULT containing the output
+     *   of the function if it succeeded, or RPC_REPLY_FAIL with the
+     *   error code if it did not.
+     *
+     * 3. Disconnect
+     *
+     *   The client terminates the connection by sending an
+     *   RPC_REQUEST_QUIT header with zero size and immediately
+     *   closing the socket.
+     */
+
     class DFHACK_EXPORT RemoteClient;
 
     class DFHACK_EXPORT RPCFunctionBase {
@@ -123,7 +160,7 @@ namespace  DFHack
         int16_t id;
     };
 
-    template<typename In, typename Out>
+    template<typename In, typename Out = EmptyMessage>
     class RemoteFunction : public RemoteFunctionBase {
     public:
         In *make_in() const { return static_cast<In*>(RemoteFunctionBase::make_in()); }
@@ -133,16 +170,33 @@ namespace  DFHack
 
         RemoteFunction() : RemoteFunctionBase(&In::default_instance(), &Out::default_instance()) {}
 
-        command_result execute(color_ostream &out) {
+        command_result execute(color_ostream &stream) {
-            return RemoteFunctionBase::execute(out, this->in(), this->out());
+            return RemoteFunctionBase::execute(stream, in(), out());
         }
-        command_result operator() (color_ostream &out, const In *input, Out *output) {
+        command_result operator() (color_ostream &stream, const In *input, Out *output) {
-            return RemoteFunctionBase::execute(out, input, output);
+            return RemoteFunctionBase::execute(stream, input, output);
+        }
+    };
+
+    template<typename In>
+    class RemoteFunction<In,EmptyMessage> : public RemoteFunctionBase {
+    public:
+        In *make_in() const { return static_cast<In*>(RemoteFunctionBase::make_in()); }
+        In *in() { return static_cast<In*>(RemoteFunctionBase::in()); }
+
+        RemoteFunction() : RemoteFunctionBase(&In::default_instance(), &EmptyMessage::default_instance()) {}
+
+        command_result execute(color_ostream &stream) {
+            return RemoteFunctionBase::execute(stream, in(), out());
+        }
+        command_result operator() (color_ostream &stream, const In *input) {
+            return RemoteFunctionBase::execute(stream, input, out());
         }
     };
 
     bool readFullBuffer(CSimpleSocket &socket, void *buf, int size);
-    bool sendRemoteMessage(CSimpleSocket &socket, int16_t id, const ::google::protobuf::MessageLite *msg);
+    bool sendRemoteMessage(CSimpleSocket &socket, int16_t id,
+                           const ::google::protobuf::MessageLite *msg, int *psz = NULL);
 
     class DFHACK_EXPORT RemoteClient
     {
@@ -160,10 +214,14 @@ namespace  DFHack
         bool connect(int port = -1);
         void disconnect();
 
+        command_result run_command(color_ostream &out, const std::string &cmd,
+                                   const std::vector<std::string> &args);
+
     private:
         bool active;
         CActiveSocket socket;
 
         RemoteFunction<dfproto::CoreBindRequest,dfproto::CoreBindReply> bind_call;
+        RemoteFunction<dfproto::CoreRunCommandRequest> runcmd_call;
     };
 }

library/include/RemoteServer.h

@@ -72,6 +72,23 @@ namespace  DFHack
         function_type fptr;
     };
 
+    template<typename In>
+    class VoidServerFunction : public ServerFunctionBase {
+    public:
+        typedef command_result (*function_type)(color_ostream &out, const In *input);
+
+        In *in() { return static_cast<In*>(RPCFunctionBase::in()); }
+
+        VoidServerFunction(RPCService *owner, const char *name, function_type fptr)
+            : ServerFunctionBase(&In::default_instance(), &EmptyMessage::default_instance(), owner, name),
+              fptr(fptr) {}
+
+        virtual command_result execute(color_ostream &stream) { return fptr(stream, in()); }
+
+    private:
+        function_type fptr;
+    };
+
     template<typename Svc, typename In, typename Out>
     class ServerMethod : public ServerFunctionBase {
     public:
@@ -92,6 +109,25 @@ namespace  DFHack
         function_type fptr;
     };
 
+    template<typename Svc, typename In>
+    class VoidServerMethod : public ServerFunctionBase {
+    public:
+        typedef command_result (Svc::*function_type)(color_ostream &out, const In *input);
+
+        In *in() { return static_cast<In*>(RPCFunctionBase::in()); }
+
+        VoidServerMethod(RPCService *owner, const char *name, function_type fptr)
+            : ServerFunctionBase(&In::default_instance(), &EmptyMessage::default_instance(), owner, name),
+              fptr(fptr) {}
+
+        virtual command_result execute(color_ostream &stream) {
+            return (static_cast<Svc*>(owner)->*fptr)(stream, in());
+        }
+
+    private:
+        function_type fptr;
+    };
+
     class Plugin;
 
     class DFHACK_EXPORT RPCService {
@@ -120,6 +156,15 @@ namespace  DFHack
             functions.push_back(new ServerFunction<In,Out>(this, name, fptr));
         }
 
+        template<typename In>
+        void addFunction(
+            const char *name,
+            command_result (*fptr)(color_ostream &out, const In *input)
+        ) {
+            assert(!owner);
+            functions.push_back(new VoidServerFunction<In>(this, name, fptr));
+        }
+
     protected:
         ServerConnection *connection() { return owner; }
 
@@ -131,6 +176,15 @@ namespace  DFHack
             assert(!owner);
             functions.push_back(new ServerMethod<Svc,In,Out>(this, name, fptr));
         }
+
+        template<typename Svc, typename In>
+        void addMethod(
+            const char *name,
+            command_result (Svc::*fptr)(color_ostream &out, const In *input)
+        ) {
+            assert(!owner);
+            functions.push_back(new VoidServerMethod<Svc,In>(this, name, fptr));
+        }
     };
 
     class CoreService : public RPCService {
@@ -141,8 +195,7 @@ namespace  DFHack
                                   const dfproto::CoreBindRequest *in,
                                   dfproto::CoreBindReply *out);
         command_result RunCommand(color_ostream &stream,
-                                  const dfproto::CoreRunCommandRequest *in,
+                                  const dfproto::CoreRunCommandRequest *in);
-                                  CoreVoidReply*);
     };
 
     class DFHACK_EXPORT ServerConnection {

library/proto/CoreProtocol.proto

@@ -42,7 +42,7 @@ message CoreErrorNotification {
     required ErrorCode code = 1;
 }
 
-message CoreVoidReply {}
+message EmptyMessage {}
 
 message CoreBindRequest {
     required string method = 1;

package/linux/dfhack

@@ -35,7 +35,7 @@ fi
 
 # Now run
 
-export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:"./stonesense/deplibs":"./hack/deplibs"
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:"./stonesense/deplibs":"./hack"
 
 case "$1" in
   -g | --gdb)

package/linux/dfhack-run

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+DF_DIR=$(dirname "$0")
+cd "${DF_DIR}"
+
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:"./stonesense/deplibs":"./hack"
+
+exec hack/dfhack-run "$@"

package/linux/egghack

@@ -1,7 +1,7 @@
 #!/bin/sh
 DF_DIR=$(dirname "$0")
 cd "${DF_DIR}"
-export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:"./stonesense/deplibs":"./hack/deplibs"
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:"./stonesense/deplibs":"./hack"
 export SDL_DISABLE_LOCK_KEYS=1 # Work around for bug in Debian/Ubuntu SDL patch.
 #export SDL_VIDEO_CENTERED=1 # Centre the screen.  Messes up resizing.
 ./libs/Dwarf_Fortress $* # Go, go, go! :)

plugins/server/CMakeLists.txt

@@ -15,7 +15,7 @@ IF(UNIX)
             ${include_directories}
             ${server_SOURCE_DIR}/zeromq
         )
-        install(PROGRAMS ${server_SOURCE_DIR}/zeromq/libzmq.so.1 DESTINATION "hack/deplibs")
+        install(PROGRAMS ${server_SOURCE_DIR}/zeromq/libzmq.so.1 DESTINATION ${DFHACK_LIBRARY_DESTINATION})
     ELSE()
         SET(PROJECT_LIBS
             zmq
@@ -31,7 +31,7 @@ ELSE()
             ${include_directories}
             ${server_SOURCE_DIR}/zeromq
         )
-        install(PROGRAMS ${server_SOURCE_DIR}/zeromq/libzmq.dll DESTINATION ".")
+        install(PROGRAMS ${server_SOURCE_DIR}/zeromq/libzmq.dll DESTINATION ${DFHACK_LIBRARY_DESTINATION})
 
 ENDIF()