Add a ListMaterials remote call for bulk download of basic material info.

library/CMakeLists.txt

@@ -218,8 +218,7 @@ ENDIF()
 ADD_LIBRARY(dfhack SHARED ${PROJECT_SOURCES})
 ADD_DEPENDENCIES(dfhack generate_headers)
 
-ADD_LIBRARY(dfhack-client SHARED RemoteClient.cpp ColorText.cpp MiscUtils.cpp
+ADD_LIBRARY(dfhack-client SHARED RemoteClient.cpp ColorText.cpp MiscUtils.cpp ${PROJECT_PROTO_SRCS})
-                                 proto/CoreProtocol.pb.cc)
 ADD_DEPENDENCIES(dfhack-client dfhack)
 
 ADD_EXECUTABLE(dfhack-run dfhack-run.cpp)

library/RemoteClient.cpp

@@ -112,7 +112,7 @@ RemoteClient::~RemoteClient()
         delete p_default_output;
 }
 
-bool DFHack::readFullBuffer(CSimpleSocket *socket, void *buf, int size)
+bool readFullBuffer(CSimpleSocket *socket, void *buf, int size)
 {
     if (!socket->IsSocketValid())
         return false;
@@ -324,14 +324,11 @@ bool RemoteFunctionBase::bind(color_ostream &out, RemoteClient *client,
     return client->bind(out, this, name, proto);
 }
 
-bool DFHack::sendRemoteMessage(CSimpleSocket *socket, int16_t id, const MessageLite *msg, int *psz)
+bool sendRemoteMessage(CSimpleSocket *socket, int16_t id, const MessageLite *msg, bool size_ready)
 {
-    int size = msg->ByteSize();
+    int size = size_ready ? msg->GetCachedSize() : msg->ByteSize();
     int fullsz = size + sizeof(RPCMessageHeader);
 
-    if (psz)
-        *psz = size;
-
     std::auto_ptr<uint8_t> data(new uint8_t[fullsz]);
     RPCMessageHeader *hdr = (RPCMessageHeader*)data.get();
 
@@ -361,7 +358,16 @@ command_result RemoteFunctionBase::execute(color_ostream &out,
         return CR_LINK_FAILURE;
     }
 
-    if (!sendRemoteMessage(p_client->socket, id, input))
+    int send_size = input->ByteSize();
+
+    if (send_size > RPCMessageHeader::MAX_MESSAGE_SIZE)
+    {
+        out.printerr("In call to %s::%s: message too large: %d.\n",
+                     this->proto.c_str(), this->name.c_str(), send_size);
+        return CR_LINK_FAILURE;
+    }
+
+    if (!sendRemoteMessage(p_client->socket, id, input, true))
     {
         out.printerr("In call to %s::%s: I/O error in send.\n",
                      this->proto.c_str(), this->name.c_str());
@@ -388,7 +394,7 @@ command_result RemoteFunctionBase::execute(color_ostream &out,
         if (header.id == RPC_REPLY_FAIL)
             return header.size == CR_OK ? CR_FAILURE : command_result(header.size);
 
-        if (header.size < 0 || header.size > 2*1048576)
+        if (header.size < 0 || header.size > RPCMessageHeader::MAX_MESSAGE_SIZE)
         {
             out.printerr("In call to %s::%s: invalid received size %d.\n",
                          this->proto.c_str(), this->name.c_str(), header.size);

library/RemoteServer.cpp

@@ -67,6 +67,11 @@ using dfproto::CoreTextNotification;
 using dfproto::CoreTextFragment;
 using google::protobuf::MessageLite;
 
+bool readFullBuffer(CSimpleSocket *socket, void *buf, int size);
+bool sendRemoteMessage(CSimpleSocket *socket, int16_t id,
+                        const ::google::protobuf::MessageLite *msg, bool size_ready);
+
+
 RPCService::RPCService()
 {
     owner = NULL;
@@ -183,7 +188,7 @@ void ServerConnection::connection_ostream::flush_proxy()
 
     buffer.clear();
 
-    if (!sendRemoteMessage(owner->socket, RPC_REPLY_TEXT, &msg))
+    if (!sendRemoteMessage(owner->socket, RPC_REPLY_TEXT, &msg, false))
     {
         owner->in_error = true;
         Core::printerr("Error writing text into client socket.\n");
@@ -243,7 +248,7 @@ void ServerConnection::threadFn(void *arg)
         if (header.id == RPC_REQUEST_QUIT)
             break;
 
-        if (header.size < 0 || header.size > 2*1048576)
+        if (header.size < 0 || header.size > RPCMessageHeader::MAX_MESSAGE_SIZE)
         {
             out.printerr("In RPC server: invalid received size %d.\n", header.size);
             break;
@@ -278,6 +283,8 @@ void ServerConnection::threadFn(void *arg)
             }
             else
             {
+                buf.reset();
+
                 reply = fn->out();
                 res = fn->execute(me->stream);
             }
@@ -287,16 +294,23 @@ void ServerConnection::threadFn(void *arg)
         if (me->in_error)
             break;
 
-        me->stream.flush();
-
         //out.print("Answer %d:%d\n", res, reply);
 
         // Send reply
-        int out_size = 0;
+        int out_size = (reply ? reply->ByteSize() : 0);
+
+        if (out_size > RPCMessageHeader::MAX_MESSAGE_SIZE)
+        {
+            me->stream.printerr("In call to %s: reply too large: %d.\n",
+                                (fn ? fn->name : "UNKNOWN"), out_size);
+            res = CR_LINK_FAILURE;
+        }
+
+        me->stream.flush();
 
         if (res == CR_OK && reply)
         {
-            if (!sendRemoteMessage(me->socket, RPC_REPLY_RESULT, reply, &out_size))
+            if (!sendRemoteMessage(me->socket, RPC_REPLY_RESULT, reply, true))
             {
                 out.printerr("In RPC server: I/O error in send result.\n");
                 break;
@@ -304,9 +318,6 @@ void ServerConnection::threadFn(void *arg)
         }
         else
         {
-            if (reply)
-                out_size = reply->ByteSize();
-
             header.id = RPC_REPLY_FAIL;
             header.size = res;
 

library/RemoteTools.cpp

@@ -49,6 +49,18 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "PluginManager.h"
 #include "MiscUtils.h"
 
+#include "modules/Materials.h"
+
+#include "DataDefs.h"
+#include "df/material.h"
+#include "df/matter_state.h"
+#include "df/inorganic_raw.h"
+#include "df/creature_raw.h"
+#include "df/plant_raw.h"
+#include "df/historical_figure.h"
+
+#include "BasicApi.pb.h"
+
 #include <cstdio>
 #include <cstdlib>
 #include <sstream>
@@ -56,9 +68,9 @@ POSSIBILITY OF SUCH DAMAGE.
 #include <memory>
 
 using namespace DFHack;
+using namespace df::enums;
+using namespace dfproto;
 
-using dfproto::CoreTextNotification;
-using dfproto::CoreTextFragment;
 using google::protobuf::MessageLite;
 
 void DFHack::strVectorToRepeatedField(RepeatedPtrField<std::string> *pf,
@@ -68,6 +80,149 @@ void DFHack::strVectorToRepeatedField(RepeatedPtrField<std::string> *pf,
         *pf->Add() = vec[i];
 }
 
+void DFHack::describeMaterial(BasicMaterialInfo *info, df::material *mat,
+                              const BasicMaterialInfoMask *mask)
+{
+    info->set_token(mat->id);
+
+    if (mask && mask->flags())
+        flagarray_to_string(info->mutable_flags(), mat->flags);
+
+    info->set_name_prefix(mat->prefix);
+
+    if (!mask || mask->states_size() == 0)
+    {
+        df::matter_state state = matter_state::Solid;
+        int temp = (mask && mask->has_temperature()) ? mask->temperature() : 10015;
+
+        if (temp >= mat->heat.melting_point)
+            state = matter_state::Liquid;
+        if (temp >= mat->heat.boiling_point)
+            state = matter_state::Gas;
+
+        info->add_state_color(mat->state_color[state]);
+        info->add_state_name(mat->state_name[state]);
+        info->add_state_adj(mat->state_adj[state]);
+    }
+    else
+    {
+        for (int i = 0; i < mask->states_size(); i++)
+        {
+            info->add_state_color(mat->state_color[i]);
+            info->add_state_name(mat->state_name[i]);
+            info->add_state_adj(mat->state_adj[i]);
+        }
+    }
+
+    if (mask && mask->reaction())
+    {
+        for (size_t i = 0; i < mat->reaction_class.size(); i++)
+            info->add_reaction_class(*mat->reaction_class[i]);
+
+        for (size_t i = 0; i < mat->reaction_product.id.size(); i++)
+        {
+            auto ptr = info->add_reaction_product();
+            ptr->set_id(*mat->reaction_product.id[i]);
+            ptr->set_type(mat->reaction_product.material.mat_type[i]);
+            ptr->set_index(mat->reaction_product.material.mat_index[i]);
+        }
+    }
+}
+
+void DFHack::describeMaterial(BasicMaterialInfo *info, const MaterialInfo &mat,
+                              const BasicMaterialInfoMask *mask)
+{
+    assert(mat.isValid());
+
+    info->set_type(mat.type);
+    info->set_index(mat.index);
+
+    describeMaterial(info, mat.material, mask);
+
+    switch (mat.mode) {
+    case MaterialInfo::Inorganic:
+        info->set_token(mat.inorganic->id);
+        if (mask && mask->flags())
+            flagarray_to_string(info->mutable_inorganic_flags(), mat.inorganic->flags);
+        break;
+
+    case MaterialInfo::Creature:
+        info->set_subtype(mat.subtype);
+        if (mat.figure)
+        {
+            info->set_hfig_id(mat.index);
+            info->set_creature_id(mat.figure->race);
+        }
+        else
+            info->set_creature_id(mat.index);
+        break;
+
+    case MaterialInfo::Plant:
+        info->set_plant_id(mat.index);
+        break;
+    }
+}
+
+static void listMaterial(ListMaterialsRes *out, int type, int index, const BasicMaterialInfoMask *mask)
+{
+    MaterialInfo info(type, index);
+    if (info.isValid())
+        describeMaterial(out->add_value(), info, mask);
+}
+
+static command_result ListMaterials(color_ostream &stream,
+                                    const ListMaterialsRq *in, ListMaterialsRes *out)
+{
+    CoreSuspender suspend;
+
+    auto mask = in->has_mask() ? &in->mask() : NULL;
+
+    for (int i = 0; i < in->id_list_size(); i++)
+    {
+        auto &elt = in->id_list(i);
+        listMaterial(out, elt.type(), elt.index(), mask);
+    }
+
+    if (in->builtin())
+    {
+        for (int i = 0; i < MaterialInfo::NUM_BUILTIN; i++)
+            listMaterial(out, i, -1, mask);
+    }
+
+    if (in->inorganic())
+    {
+        auto &vec = df::inorganic_raw::get_vector();
+        for (size_t i = 0; i < vec.size(); i++)
+            listMaterial(out, 0, i, mask);
+    }
+
+    if (in->creatures())
+    {
+        auto &vec = df::creature_raw::get_vector();
+        for (size_t i = 0; i < vec.size(); i++)
+        {
+            auto praw = vec[i];
+
+            for (size_t j = 0; j < praw->material.size(); j++)
+                listMaterial(out, MaterialInfo::CREATURE_BASE+j, i, mask);
+        }
+    }
+
+    if (in->plants())
+    {
+        auto &vec = df::plant_raw::get_vector();
+        for (size_t i = 0; i < vec.size(); i++)
+        {
+            auto praw = vec[i];
+
+            for (size_t j = 0; j < praw->material.size(); j++)
+                listMaterial(out, MaterialInfo::PLANT_BASE+j, i, mask);
+        }
+    }
+
+    return out->value_size() ? CR_OK : CR_NOT_FOUND;
+}
+
 CoreService::CoreService() {
     suspend_depth = 0;
 
@@ -78,6 +233,8 @@ CoreService::CoreService() {
     // Add others here:
     addMethod("CoreSuspend", &CoreService::CoreSuspend);
     addMethod("CoreResume", &CoreService::CoreResume);
+
+    addFunction("ListMaterials", ListMaterials);
 }
 
 CoreService::~CoreService()

library/include/DataDefs.h

@@ -373,7 +373,7 @@ namespace DFHack {
      * Find a flag array item by key string. Returns success code.
      */
     template<class T>
-    inline bool find_bitfield_field(unsigned *idx, const std::string &name, const BitArray<T>*) {
+    inline bool find_flagarray_field(unsigned *idx, const std::string &name, const BitArray<T>*) {
         T tmp;
         if (!find_enum_item(&tmp, name) || tmp < 0) return false;
         *idx = unsigned(tmp);
@@ -384,7 +384,7 @@ namespace DFHack {
      * Find a flag array item by key and set its value. Returns success code.
      */
     template<class T>
-    inline bool set_bitfield_field(BitArray<T> *bitfield, const std::string &name, int value)
+    inline bool set_flagarray_field(BitArray<T> *bitfield, const std::string &name, int value)
     {
         T tmp;
         if (!find_enum_item(&tmp, name) || tmp < 0) return false;
@@ -396,7 +396,7 @@ namespace DFHack {
      * Find a flag array item by key and retrieve its value. Returns success code.
      */
     template<class T>
-    inline bool get_bitfield_field(int *value, const BitArray<T> &bitfield, const std::string &name)
+    inline bool get_flagarray_field(int *value, const BitArray<T> &bitfield, const std::string &name)
     {
         T tmp;
         if (!find_enum_item(&tmp, name) || tmp < 0) return false;
@@ -411,13 +411,22 @@ namespace DFHack {
      * Represent flag array bits as strings in a vector.
      */
     template<class T>
-    inline void bitfield_to_string(std::vector<std::string> *pvec, const BitArray<T> &val) {
+    inline void flagarray_to_string(std::vector<std::string> *pvec, const BitArray<T> &val) {
         typedef df::enum_traits<T> traits;
         int size = traits::last_item_value-traits::first_item_value+1;
         flagarrayToString(pvec, val.bits, val.size,
                           (int)traits::first_item_value, size, traits::key_table);
     }
 
+    /**
+     * Represent flag array bits as a string, using sep as join separator.
+     */
+    template<class T>
+    inline std::string bitfield_to_string(const BitArray<T> &val, const std::string &sep = " ") {
+        std::vector<std::string> tmp;
+        flagarray_to_string<T>(&tmp, val);
+        return join_strings(sep, tmp);
+    }
 }
 
 #define ENUM_ATTR(enum,attr,val) (df::enum_traits<df::enum>::attrs(val).attr)

library/include/RemoteClient.h

@@ -66,6 +66,8 @@ namespace  DFHack
     };
 
     struct RPCMessageHeader {
+        static const int MAX_MESSAGE_SIZE = 8*1048756;
+
         int16_t id;
         int32_t size;
     };
@@ -220,10 +222,6 @@ namespace  DFHack
         }
     };
 
-    bool readFullBuffer(CSimpleSocket *socket, void *buf, int size);
-    bool sendRemoteMessage(CSimpleSocket *socket, int16_t id,
-                           const ::google::protobuf::MessageLite *msg, int *psz = NULL);
-
     class DFHACK_EXPORT RemoteClient
     {
         friend class RemoteFunctionBase;

library/include/RemoteTools.h

@@ -29,8 +29,17 @@ distribution.
 
 #include "DataDefs.h"
 
+#include "Basic.pb.h"
+
+namespace df
+{
+    struct material;
+}
+
 namespace DFHack
 {
+    class MaterialInfo;
+
     using google::protobuf::RepeatedPtrField;
 
     DFHACK_EXPORT void strVectorToRepeatedField(RepeatedPtrField<std::string> *pf,
@@ -46,6 +55,27 @@ namespace DFHack
         strVectorToRepeatedField(pf, tmp);
     }
 
+    /**
+     * Represent flagarray bits as a repeated string field.
+     */
+    template<class T>
+    inline void flagarray_to_string(RepeatedPtrField<std::string> *pf, const BitArray<T> &val) {
+        std::vector<std::string> tmp;
+        flagarray_to_string<T>(&tmp, val);
+        strVectorToRepeatedField(pf, tmp);
+    }
+
+    /////
+
+    using dfproto::BasicMaterialInfo;
+    using dfproto::BasicMaterialInfoMask;
+
+    DFHACK_EXPORT void describeMaterial(BasicMaterialInfo *info, df::material *mat,
+                                        const BasicMaterialInfoMask *mask = NULL);
+    DFHACK_EXPORT void describeMaterial(BasicMaterialInfo *info, const MaterialInfo &mat,
+                                        const BasicMaterialInfoMask *mask = NULL);
+
+    /////
 
     class CoreService : public RPCService {
         int suspend_depth;

library/proto/Basic.proto

@@ -0,0 +1,54 @@
+package dfproto;
+
+option optimize_for = LITE_RUNTIME;
+
+message BasicMaterialId {
+    required int32 type = 1;
+    required sint32 index = 2;
+};
+
+message BasicMaterialInfo {
+    required int32 type = 1;
+    required sint32 index = 2;
+
+    required string token = 3;
+    repeated string flags = 4;
+
+    optional int32 subtype = 5 [default = -1];
+    optional int32 creature_id = 6 [default = -1];
+    optional int32 plant_id = 7 [default = -1];
+    optional int32 hfig_id = 8 [default = -1];
+
+    optional string name_prefix = 9 [default = ""];
+
+    repeated fixed32 state_color = 10;
+    repeated string state_name = 11;
+    repeated string state_adj = 12;
+
+    message Product {
+        required string id = 1;
+        required int32 type = 2;
+        required sint32 index = 3;
+    };
+    repeated string reaction_class = 13;
+    repeated Product reaction_product = 14;
+
+    repeated string inorganic_flags = 15;
+};
+
+message BasicMaterialInfoMask {
+    enum StateType {
+        Solid = 0;
+        Liquid = 1;
+        Gas = 2;
+        Powder = 3;
+        Paste = 4;
+        Pressed = 5;
+    };
+    repeated StateType states = 1;
+
+    optional bool flags = 2 [default = false];
+    optional bool reaction = 3 [default = false];
+    optional int32 temperature = 4;
+};
+

library/proto/BasicApi.proto

@@ -0,0 +1,17 @@
+package dfproto;
+
+option optimize_for = LITE_RUNTIME;
+
+import "Basic.proto";
+
+message ListMaterialsRq {
+    optional BasicMaterialInfoMask mask = 1;
+    repeated BasicMaterialId id_list = 2;
+    optional bool builtin = 3;
+    optional bool inorganic = 4;
+    optional bool creatures = 5;
+    optional bool plants = 6;
+};
+message ListMaterialsRes {
+    repeated BasicMaterialInfo value = 1;
+};