Merge remote-tracking branch 'myk002/quickfort_xlsxio' into develop

.gitmodules

@@ -16,3 +16,15 @@
 [submodule "depends/jsoncpp"]
 	path = depends/jsoncpp-sub
 	url = ../../DFHack/jsoncpp.git
+[submodule "depends/xlsxio"]
+	path = depends/xlsxio
+	url = ../../brechtsanders/xlsxio.git
+	shallow = true
+[submodule "depends/libzip"]
+	path = depends/libzip
+	url = ../../nih-at/libzip.git
+	shallow = true
+[submodule "depends/libexpat"]
+	path = depends/libexpat
+	url = ../../libexpat/libexpat.git
+	shallow = true

CMakeLists.txt

@@ -178,7 +178,7 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}")
 endif()
 
 # make sure all the necessary submodules have been set up
-if(NOT EXISTS ${dfhack_SOURCE_DIR}/library/xml/codegen.pl OR NOT EXISTS ${dfhack_SOURCE_DIR}/depends/clsocket/CMakeLists.txt)
+if(NOT EXISTS ${dfhack_SOURCE_DIR}/library/xml/codegen.pl OR NOT EXISTS ${dfhack_SOURCE_DIR}/depends/clsocket/CMakeLists.txt OR NOT EXISTS ${dfhack_SOURCE_DIR}/depends/libexpat/expat/CMakeLists.txt OR NOT EXISTS ${dfhack_SOURCE_DIR}/depends/libzip/CMakeLists.txt OR NOT EXISTS ${dfhack_SOURCE_DIR}/depends/xlsxio/CMakeLists.txt)
     message(SEND_ERROR "One or more required submodules could not be found! Run 'git submodule update --init' from the root DFHack directory. (See the section 'Getting the Code' in docs/Compile.rst)")
 endif()
 

depends/CMakeLists.txt

@@ -17,3 +17,62 @@ option(CLSOCKET_SHARED "Build clsocket lib as shared." OFF)
 option(CLSOCKET_DEP_ONLY "Build for use inside other CMake projects as dependency." ON)
 add_subdirectory(clsocket)
 ide_folder(clsocket "Depends")
+
+# assemble environment args to pass on to dependency projects
+get_cmake_property(vars CACHE_VARIABLES)
+foreach(var ${vars})
+  if(var MATCHES "^CMAKE_"
+     AND NOT var MATCHES "^CMAKE_CACHE"
+     AND NOT var MATCHES "^CMAKE_HOME"
+     AND NOT var MATCHES "^CMAKE_PROJECT")
+    list(APPEND CL_ARGS "-D${var}=${${var}}")
+  endif()
+  if(var MATCHES "^ZLIB")
+    list(APPEND CL_ARGS "-D${var}=${${var}}")
+  endif()
+endforeach()
+
+include(ExternalProject)
+
+if(WIN32)
+  set(EXPAT_LIB_NAME_SUFFIX "MD")
+  set(XLSXIO_C_FLAGS "${CMAKE_C_FLAGS} /DXML_STATIC")
+else()
+  set(EXPAT_LIB_NAME_SUFFIX "")
+  set(XLSXIO_C_FLAGS "${CMAKE_C_FLAGS} -DXML_STATIC")
+endif()
+
+set(LIBEXPAT_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/libexpat)
+set(LIBEXPAT_LIB ${LIBEXPAT_INSTALL_DIR}/lib/libexpat${EXPAT_LIB_NAME_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
+set(LIBEXPAT_LIB ${LIBEXPAT_LIB} PARENT_SCOPE)
+ExternalProject_Add(libexpat_project
+    PREFIX      libexpat
+    SOURCE_DIR  ${CMAKE_CURRENT_SOURCE_DIR}/libexpat/expat
+    INSTALL_DIR ${LIBEXPAT_INSTALL_DIR}
+    BUILD_BYPRODUCTS ${LIBEXPAT_LIB}
+    CMAKE_ARGS  ${CL_ARGS} -DCMAKE_INSTALL_PREFIX=${LIBEXPAT_INSTALL_DIR} -DCMAKE_BUILD_TYPE=Release -DEXPAT_BUILD_EXAMPLES=OFF -DEXPAT_BUILD_TESTS=OFF -DEXPAT_BUILD_TOOLS=OFF -DEXPAT_SHARED_LIBS=OFF -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+)
+
+set(LIBZIP_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/libzip)
+set(LIBZIP_LIB ${LIBZIP_INSTALL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}zip${CMAKE_STATIC_LIBRARY_SUFFIX})
+set(LIBZIP_LIB ${LIBZIP_LIB} PARENT_SCOPE)
+ExternalProject_Add(libzip_project
+    PREFIX      libzip
+    SOURCE_DIR  ${CMAKE_CURRENT_SOURCE_DIR}/libzip
+    INSTALL_DIR ${LIBZIP_INSTALL_DIR}
+    BUILD_BYPRODUCTS ${LIBZIP_LIB}
+    CMAKE_ARGS  ${CL_ARGS} -DCMAKE_INSTALL_PREFIX=${LIBZIP_INSTALL_DIR} -DCMAKE_BUILD_TYPE=Release -DBUILD_DOC=OFF -DBUILD_EXAMPLES=OFF -DBUILD_REGRESS=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_TOOLS=OFF -DENABLE_BZIP2=OFF -DENABLE_COMMONCRYPTO=OFF -DENABLE_GNUTLS=OFF -DENABLE_LZMA=OFF -DENABLE_MBEDTLS=OFF -DENABLE_OPENSSL=OFF -DENABLE_WINDOWS_CRYPTO=OFF -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+)
+
+set(XLSXIO_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/xlsxio)
+set(XLSXIO_INSTALL_DIR ${XLSXIO_INSTALL_DIR} PARENT_SCOPE)
+set(XLSXIO_LIB ${XLSXIO_INSTALL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}xlsxio_read${CMAKE_STATIC_LIBRARY_SUFFIX})
+set(XLSXIO_LIB ${XLSXIO_LIB} PARENT_SCOPE)
+ExternalProject_Add(xlsxio_project
+    PREFIX      xlsxio
+    DEPENDS     libexpat_project libzip_project
+    SOURCE_DIR  ${CMAKE_CURRENT_SOURCE_DIR}/xlsxio
+    INSTALL_DIR ${XLSXIO_INSTALL_DIR}
+    BUILD_BYPRODUCTS ${XLSXIO_LIB}
+    CMAKE_ARGS  ${CL_ARGS} -DCMAKE_INSTALL_PREFIX=${XLSXIO_INSTALL_DIR} -DCMAKE_BUILD_TYPE=Release -DBUILD_STATIC=ON -DBUILD_SHARED=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_TOOLS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_DOCUMENTATION=OFF -DWITH_LIBZIP=ON -DZLIB_DIR=${ZLIB_DIR} -DLIBZIP_DIR=${LIBZIP_INSTALL_DIR} -DEXPAT_DIR=${LIBEXPAT_INSTALL_DIR} -DEXPAT_LIBRARIES=${LIBEXPAT_LIB} -DEXPAT_INCLUDE_DIRS=${LIBEXPAT_INSTALL_DIR}/include -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_C_FLAGS=${XLSXIO_C_FLAGS}
+)

depends/libexpat

@@ -0,0 +1 @@
+Subproject commit a7bc26b69768f7fb24f0c7976fae24b157b85b13

depends/libzip

@@ -0,0 +1 @@
+Subproject commit 29b881d286f43189ff7392f609da78daa80c0606

depends/xlsxio

@@ -0,0 +1 @@
+Subproject commit 261d56815b29908fc960fecb9cb3143db4b485ad

docs/Lua API.rst

@@ -4102,6 +4102,51 @@ Lua plugin classes
 - ``shuffle()``: shuffles the sequence of numbers
 - ``next()``: returns next number in the sequence
 
+.. _xlsxreader:
+
+xlsxreader
+==========
+
+Utility functions to facilitate reading .xlsx spreadsheets. It provides the
+following API methods:
+
+ - ``file_handle open_xlsx_file(filename)``
+ - ``close_xlsx_file(file_handle)``
+ - ``sheet_names list_sheets(file_handle)``
+ - ``sheet_handle open_sheet(file_handle, sheet_name)``
+ - ``close_sheet(sheet_handle)``
+ - ``cell_strings get_row(sheet_handle)``
+
+ Example::
+
+    local xlsxreader = require('plugins.xlsxreader')
+
+    local function dump_sheet(xlsx_file, sheet_name)
+        print('reading sheet: '..sheet_name)
+        local xlsx_sheet = xlsxreader.open_sheet(xlsx_file, sheet_name)
+        dfhack.with_finalize(
+            function () xlsxreader.close_sheet(xlsx_sheet) end,
+            function ()
+                local row_cells = xlsxreader.get_row(xlsx_sheet)
+                while row_cells do
+                    printall(row_cells)
+                    row_cells = xlsxreader.get_row(xlsx_sheet)
+                end
+            end
+        )
+    end
+
+    local filepath = "path/to/some_file.xlsx"
+    local xlsx_file = xlsxreader.open_xlsx_file(filepath)
+    dfhack.with_finalize(
+        function () xlsxreader.close_xlsx_file(xlsx_file) end,
+        function ()
+            for _, sheet_name in ipairs(xlsxreader.list_sheets(xlsx_file)) do
+                dump_sheet(xlsx_file, sheet_name)
+            end
+        end
+    )
+
 =======
 Scripts
 =======

docs/Plugins.rst

@@ -2946,3 +2946,4 @@ in the `lua-api` file under `lua-plugins`:
 * `luasocket`
 * `map-render`
 * `cxxrandom`
+* `xlsxreader`

library/CMakeLists.txt

@@ -40,6 +40,8 @@ set(MAIN_HEADERS
     include/Module.h
     include/Pragma.h
     include/MemAccess.h
+    include/PluginManager.h
+    include/PluginStatics.h
     include/Signal.hpp
     include/TileTypes.h
     include/Types.h
@@ -72,6 +74,7 @@ set(MAIN_SOURCES
     MiscUtils.cpp
     Types.cpp
     PluginManager.cpp
+    PluginStatics.cpp
     TileTypes.cpp
     VersionInfoFactory.cpp
     RemoteClient.cpp

library/PluginStatics.cpp

@@ -0,0 +1,9 @@
+#include "PluginStatics.h"
+
+namespace DFHack {
+
+// xlsxreader statics
+DFHACK_EXPORT xlsx_file_handle_identity xlsx_file_handle::_identity;
+DFHACK_EXPORT xlsx_sheet_handle_identity xlsx_sheet_handle::_identity;
+
+}

library/include/PluginStatics.h

@@ -0,0 +1,42 @@
+/*
+ * This file and the companion PluginStatics.cpp contain static structures used
+ * by DFHack plugins. Linking them here, into the dfhack library, instead of
+ * into the plugins themselves allows the plugins to be freely unloaded and
+ * reloaded without fear of causing cached references to static data becoming
+ * corrupted.
+ */
+
+#pragma once
+
+#include "DataIdentity.h"
+
+namespace DFHack {
+
+// xlsxreader definitions
+struct DFHACK_EXPORT xlsx_file_handle_identity : public compound_identity {
+    xlsx_file_handle_identity()
+        :compound_identity(0, nullptr, nullptr, "xlsx_file_handle") {};
+    DFHack::identity_type type() override { return IDTYPE_OPAQUE; }
+};
+
+struct DFHACK_EXPORT xlsx_sheet_handle_identity : public compound_identity {
+    xlsx_sheet_handle_identity()
+        :compound_identity(0, nullptr, nullptr, "xlsx_sheet_handle") {};
+    DFHack::identity_type type() override { return IDTYPE_OPAQUE; }
+};
+
+struct DFHACK_EXPORT xlsx_file_handle {
+    typedef struct xlsxio_read_struct* xlsxioreader;
+    const xlsxioreader handle;
+    xlsx_file_handle(xlsxioreader handle): handle(handle) {}
+    static xlsx_file_handle_identity _identity;
+};
+
+struct DFHACK_EXPORT xlsx_sheet_handle {
+    typedef struct xlsxio_read_sheet_struct* xlsxioreadersheet;
+    const xlsxioreadersheet handle;
+    xlsx_sheet_handle(xlsxioreadersheet handle): handle(handle) {}
+    static xlsx_sheet_handle_identity _identity;
+};
+
+}

plugins/CMakeLists.txt

@@ -77,6 +77,20 @@ set_source_files_properties( Brushes.h PROPERTIES HEADER_FILE_ONLY TRUE )
 add_library(buildingplan-lib STATIC buildingplan-lib.cpp)
 target_link_libraries(buildingplan-lib dfhack)
 
+# xlsxreader deps
+add_library(xlsxio_read STATIC IMPORTED)
+set_target_properties(xlsxio_read PROPERTIES IMPORTED_LOCATION ${XLSXIO_LIB})
+add_library(zip STATIC IMPORTED)
+set_target_properties(zip PROPERTIES IMPORTED_LOCATION ${LIBZIP_LIB})
+add_library(expat STATIC IMPORTED)
+set_target_properties(expat PROPERTIES IMPORTED_LOCATION ${LIBEXPAT_LIB})
+
+if(WIN32)
+  set(LIB_Z_LIB "depends/zlib/lib/zlib")
+else()
+  set(LIB_Z_LIB "z")
+endif()
+
 # Plugins
 option(BUILD_SUPPORTED "Build the supported plugins (reveal, probe, etc.)." ON)
 if(BUILD_SUPPORTED)
@@ -175,8 +189,11 @@ if(BUILD_SUPPORTED)
     add_subdirectory(tweak)
     dfhack_plugin(workflow workflow.cpp LINK_LIBRARIES lua)
     dfhack_plugin(workNow workNow.cpp)
+    dfhack_plugin(xlsxreader xlsxreader.cpp LINK_LIBRARIES lua xlsxio_read zip expat ${LIB_Z_LIB})
     dfhack_plugin(zone zone.cpp LINK_LIBRARIES lua)
 endif()
+target_include_directories(xlsxreader PRIVATE ${XLSXIO_INSTALL_DIR}/include)
+add_dependencies(xlsxreader xlsxio_project)
 
 # this is the skeleton plugin. If you want to make your own, make a copy and then change it
 option(BUILD_SKELETON "Build the skeleton plugin." OFF)

plugins/lua/xlsxreader.lua

@@ -0,0 +1,47 @@
+local _ENV = mkmodule('plugins.xlsxreader')
+
+--[[
+
+ Native functions:
+
+ * file_handle open_xlsx_file(filename)
+ * close_xlsx_file(file_handle)
+ * sheet_names list_sheets(file_handle)
+
+ * sheet_handle open_sheet(file_handle, sheet_name)
+ * close_sheet(sheet_handle)
+ * cell_strings get_row(sheet_handle)
+
+Sample usage:
+
+    local xlsxreader = require('plugins.xlsxreader')
+
+    local function dump_sheet(xlsx_file, sheet_name)
+        print('reading sheet: '..sheet_name)
+        local xlsx_sheet = xlsxreader.open_sheet(xlsx_file, sheet_name)
+        dfhack.with_finalize(
+            function () xlsxreader.close_sheet(xlsx_sheet) end,
+            function ()
+                local row_cells = xlsxreader.get_row(xlsx_sheet)
+                while row_cells do
+                    printall(row_cells)
+                    row_cells = xlsxreader.get_row(xlsx_sheet)
+                end
+            end
+        )
+    end
+
+    local filepath = "path/to/some_file.xlsx"
+    local xlsx_file = xlsxreader.open_xlsx_file(filepath)
+    dfhack.with_finalize(
+        function () xlsxreader.close_xlsx_file(xlsx_file) end,
+        function ()
+            for _, sheet_name in ipairs(xlsxreader.list_sheets(xlsx_file)) do
+                dump_sheet(xlsx_file, sheet_name)
+            end
+        end
+    )
+
+--]]
+
+return _ENV

plugins/xlsxreader.cpp

@@ -0,0 +1,145 @@
+/*
+ * Wrapper for xlsxio_read library functions.
+ */
+
+#define BUILD_XLSXIO_STATIC
+#include <xlsxio_read.h>
+
+#include "DataFuncs.h"
+#include "LuaTools.h"
+#include "PluginManager.h"
+#include "PluginStatics.h"
+
+using namespace DFHack;
+
+DFHACK_PLUGIN("xlsxreader");
+
+// returns NULL if the file failed to open
+xlsx_file_handle* open_xlsx_file(std::string filename) {
+    xlsxioreader opaque_file_handle = xlsxioread_open(filename.c_str());
+    if (!opaque_file_handle) {
+        return NULL;
+    }
+    return new xlsx_file_handle(opaque_file_handle);
+}
+
+void close_xlsx_file(xlsx_file_handle *file_handle) {
+    if (!file_handle) {
+        return;
+    }
+    if (file_handle->handle) {
+        xlsxioread_close(file_handle->handle);
+    }
+    delete(file_handle);
+}
+
+// returns XLSXIOReaderSheet object or NULL on error
+xlsx_sheet_handle* open_sheet(xlsx_file_handle *file_handle,
+                              std::string sheet_name) {
+    CHECK_NULL_POINTER(file_handle);
+    CHECK_NULL_POINTER(file_handle->handle);
+    xlsxioreadersheet opaque_sheet_handle = xlsxioread_sheet_open(
+        file_handle->handle, sheet_name.c_str(), XLSXIOREAD_SKIP_NONE);
+    if (!opaque_sheet_handle) {
+        return NULL;
+    }
+    return new xlsx_sheet_handle(opaque_sheet_handle);
+}
+
+void close_sheet(xlsx_sheet_handle *sheet_handle) {
+    if (!sheet_handle) {
+        return;
+    }
+    if (sheet_handle->handle) {
+        xlsxioread_sheet_close(sheet_handle->handle);
+    }
+    delete(sheet_handle);
+}
+
+static int list_callback(const XLSXIOCHAR* name, void* cbdata) {
+    auto sheetNames = (std::vector<std::string> *)cbdata;
+    sheetNames->push_back(name);
+    return 0;
+}
+
+// start reading the next row of data; must be called before get_next_cell.
+// returns false if there is no next row to get.
+static bool get_next_row(xlsx_sheet_handle *sheet_handle) {
+    return xlsxioread_sheet_next_row(sheet_handle->handle) != 0;
+}
+
+// fills the value param with the contents of the cell in the next column cell
+// in the current row.
+// returns false if there are no more cells in this row.
+static bool get_next_cell(xlsx_sheet_handle *sheet_handle, std::string& value) {
+    char* result;
+    if (!xlsxioread_sheet_next_cell_string(sheet_handle->handle, &result)) {
+        value.clear();
+        return false;
+    }
+    value.assign(result);
+    free(result);
+    return true;
+}
+
+// internal function to extract a handle from the first stack param
+static void * get_xlsxreader_handle(lua_State *L) {
+    if (lua_gettop(L) < 1 || lua_isnil(L, 1)) {
+        luaL_error(L, "invalid xlsxreader handle");
+    }
+    luaL_checktype(L, 1, LUA_TUSERDATA);
+    return LuaWrapper::get_object_ref(L, 1);
+}
+
+// takes a file handle and returns a table-list of sheet names
+int list_sheets(lua_State *L) {
+    auto file_handle = (xlsx_file_handle *)get_xlsxreader_handle(L);
+    CHECK_NULL_POINTER(file_handle->handle);
+    auto sheetNames = std::vector<std::string>();
+    xlsxioread_list_sheets(file_handle->handle, list_callback, &sheetNames);
+    Lua::PushVector(L, sheetNames, true);
+    return 1;
+}
+
+// takes the sheet handle and returns a table-list of strings, or nil if we
+// already processed the last row in the file.
+int get_row(lua_State *L) {
+    auto sheet_handle = (xlsx_sheet_handle *)get_xlsxreader_handle(L);
+    CHECK_NULL_POINTER(sheet_handle->handle);
+    bool ok = get_next_row(sheet_handle);
+    if (!ok) {
+        lua_pushnil(L);
+    } else {
+        std::string value;
+        auto cells = std::vector<std::string>();
+        while (get_next_cell(sheet_handle, value)) {
+            cells.push_back(value);
+        }
+        Lua::PushVector(L, cells, true);
+    }
+
+    return 1;
+}
+
+DFHACK_PLUGIN_LUA_FUNCTIONS {
+    DFHACK_LUA_FUNCTION(open_xlsx_file),
+    DFHACK_LUA_FUNCTION(close_xlsx_file),
+    DFHACK_LUA_FUNCTION(open_sheet),
+    DFHACK_LUA_FUNCTION(close_sheet),
+    DFHACK_LUA_END
+};
+
+DFHACK_PLUGIN_LUA_COMMANDS{
+    DFHACK_LUA_COMMAND(list_sheets),
+    DFHACK_LUA_COMMAND(get_row),
+    DFHACK_LUA_END
+};
+
+DFhackCExport command_result plugin_init(color_ostream &,
+                                         std::vector<PluginCommand> &) {
+    return CR_OK;
+}
+
+DFhackCExport command_result plugin_shutdown(color_ostream &) {
+    return CR_OK;
+}