From 69a2f440201eb3b7b2842ee0b2f76758962bbbf1 Mon Sep 17 00:00:00 2001 From: myk002 Date: Mon, 8 Mar 2021 15:04:50 -0800 Subject: [PATCH 1/6] Lua class wrappers for the xlsxreader plugin API --- docs/Lua API.rst | 35 ++++++++------- plugins/lua/xlsxreader.lua | 92 +++++++++++++++++++++----------------- plugins/xlsxreader.cpp | 8 +++- 3 files changed, 75 insertions(+), 60 deletions(-) diff --git a/docs/Lua API.rst b/docs/Lua API.rst index 764e56dd6..c48d522f6 100644 --- a/docs/Lua API.rst +++ b/docs/Lua API.rst @@ -4305,41 +4305,44 @@ xlsxreader ========== Utility functions to facilitate reading .xlsx spreadsheets. It provides the -following API methods: +following low-level API methods: - ``file_handle open_xlsx_file(filename)`` - ``close_xlsx_file(file_handle)`` - ``sheet_names list_sheets(file_handle)`` - ``sheet_handle open_sheet(file_handle, sheet_name)`` - ``close_sheet(sheet_handle)`` - - ``cell_strings get_row(sheet_handle)`` + - ``cell_strings get_row(sheet_handle, max_tokens)`` The ``max_tokens`` + parameter is optional. If set to a number > 0, it limits the number of cells + read and returned for the row. - Example:: + Lua users would be better off using the Lua class wrappers, though. For + example:: local xlsxreader = require('plugins.xlsxreader') - local function dump_sheet(xlsx_file, sheet_name) - print('reading sheet: '..sheet_name) - local xlsx_sheet = xlsxreader.open_sheet(xlsx_file, sheet_name) + local function dump_sheet(reader, sheet_name) + print('reading sheet: ' .. sheet_name) + local sheet_reader = reader:open_sheet(sheet_name) dfhack.with_finalize( - function () xlsxreader.close_sheet(xlsx_sheet) end, - function () - local row_cells = xlsxreader.get_row(xlsx_sheet) + function() sheet_reader:close() end, + function() + local row_cells = sheet_reader:get_row() while row_cells do printall(row_cells) - row_cells = xlsxreader.get_row(xlsx_sheet) + row_cells = sheet_reader:get_row() end end ) end - local filepath = "path/to/some_file.xlsx" - local xlsx_file = xlsxreader.open_xlsx_file(filepath) + local filepath = 'path/to/some_file.xlsx' + local reader = xlsxreader.XlsxioReader{filepath=filepath} dfhack.with_finalize( - function () xlsxreader.close_xlsx_file(xlsx_file) end, - function () - for _, sheet_name in ipairs(xlsxreader.list_sheets(xlsx_file)) do - dump_sheet(xlsx_file, sheet_name) + function() reader:close() end, + function() + for _,sheet_name in ipairs(reader:list_sheets()) do + dump_sheet(reader, sheet_name) end end ) diff --git a/plugins/lua/xlsxreader.lua b/plugins/lua/xlsxreader.lua index 323231915..b9827c39f 100644 --- a/plugins/lua/xlsxreader.lua +++ b/plugins/lua/xlsxreader.lua @@ -1,47 +1,55 @@ local _ENV = mkmodule('plugins.xlsxreader') ---[[ - - Native functions: - - * file_handle open_xlsx_file(filename) - * close_xlsx_file(file_handle) - * sheet_names list_sheets(file_handle) - - * sheet_handle open_sheet(file_handle, sheet_name) - * close_sheet(sheet_handle) - * cell_strings get_row(sheet_handle) - -Sample usage: - - local xlsxreader = require('plugins.xlsxreader') - - local function dump_sheet(xlsx_file, sheet_name) - print('reading sheet: '..sheet_name) - local xlsx_sheet = xlsxreader.open_sheet(xlsx_file, sheet_name) - dfhack.with_finalize( - function () xlsxreader.close_sheet(xlsx_sheet) end, - function () - local row_cells = xlsxreader.get_row(xlsx_sheet) - while row_cells do - printall(row_cells) - row_cells = xlsxreader.get_row(xlsx_sheet) - end - end - ) +XlsxioSheetReader = defclass(XlsxioSheetReader, nil) +XlsxioSheetReader.ATTRS{ + -- handle to the open sheet. required. + sheet_handle = DEFAULT_NIL, +} + +function XlsxioSheetReader:init() + if not self.sheet_handle then + error('XlsxSheetReader: sheet_handle is required') end - - local filepath = "path/to/some_file.xlsx" - local xlsx_file = xlsxreader.open_xlsx_file(filepath) - dfhack.with_finalize( - function () xlsxreader.close_xlsx_file(xlsx_file) end, - function () - for _, sheet_name in ipairs(xlsxreader.list_sheets(xlsx_file)) do - dump_sheet(xlsx_file, sheet_name) - end - end - ) - ---]] +end + +function XlsxioSheetReader:close() + close_sheet(self.sheet_handle) + self.sheet_handle = nil +end + +function XlsxioSheetReader:get_row(max_tokens) + return get_row(self.sheet_handle, max_tokens) +end + +XlsxioReader = defclass(XlsxioReader, nil) +XlsxioReader.ATTRS{ + -- full or relative path to the target .xlsx file. required. + filepath = DEFAULT_NIL, +} + +function XlsxioReader:init() + if not self.filepath then + error('XlsxReader: filepath is required') + end + self.xlsx_handle = open_xlsx_file(self.filepath) + if not self.xlsx_handle then + qerror(('failed to open "%s"'):format(self.filepath)) + end +end + +function XlsxioReader:close() + close_xlsx_file(self.xlsx_handle) + self.xlsx_handle = nil +end + +function XlsxioReader:list_sheets() + return list_sheets(self.xlsx_handle) +end + +-- if sheet_name is empty or nil, opens the first sheet +function XlsxioReader:open_sheet(sheet_name) + local sheet_handle = open_sheet(self.xlsx_handle, sheet_name) + return XlsxioSheetReader{sheet_handle=sheet_handle} +end return _ENV diff --git a/plugins/xlsxreader.cpp b/plugins/xlsxreader.cpp index 7f61d6ff5..d9a781f11 100644 --- a/plugins/xlsxreader.cpp +++ b/plugins/xlsxreader.cpp @@ -101,11 +101,13 @@ int list_sheets(lua_State *L) { return 1; } -// takes the sheet handle and returns a table-list of strings, or nil if we -// already processed the last row in the file. +// takes the sheet handle and returns a table-list of strings, limited to the +// requested number of tokens if specified and > 0, or nil if we already +// processed the last row in the file. int get_row(lua_State *L) { auto sheet_handle = (xlsx_sheet_handle *)get_xlsxreader_handle(L); CHECK_NULL_POINTER(sheet_handle->handle); + lua_Integer max_tokens = luaL_optinteger(L, 2, 0); bool ok = get_next_row(sheet_handle); if (!ok) { lua_pushnil(L); @@ -114,6 +116,8 @@ int get_row(lua_State *L) { auto cells = std::vector(); while (get_next_cell(sheet_handle, value)) { cells.push_back(value); + if (max_tokens > 0 && cells.size() >= max_tokens) + break; } Lua::PushVector(L, cells, true); } From f3bef8c34e587dc7f6f2087afbf2352a284efc0e Mon Sep 17 00:00:00 2001 From: myk002 Date: Mon, 8 Mar 2021 15:06:27 -0800 Subject: [PATCH 2/6] update changelog --- docs/changelog.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.txt b/docs/changelog.txt index cece75f72..e94d20266 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -33,6 +33,9 @@ changelog.txt uses a syntax similar to RST, with a few special sequences: # Future +## Lua +- `xlsxreader`: Added Lua class wrappers for the xlsxreader plugin API + ## Documentation - Added more client library implementations to the `remote interface docs ` From aac958aa5060564e003c7355b5b4ad74dee54dcc Mon Sep 17 00:00:00 2001 From: myk002 Date: Mon, 22 Mar 2021 09:38:20 -0700 Subject: [PATCH 3/6] add open() wrapper fn and document class methods --- docs/Lua API.rst | 41 +++++++++++++++++++++++++++++--------- plugins/lua/xlsxreader.lua | 4 ++++ 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/docs/Lua API.rst b/docs/Lua API.rst index c48d522f6..83eef2956 100644 --- a/docs/Lua API.rst +++ b/docs/Lua API.rst @@ -4307,17 +4307,40 @@ xlsxreader Utility functions to facilitate reading .xlsx spreadsheets. It provides the following low-level API methods: - - ``file_handle open_xlsx_file(filename)`` + - ``open_xlsx_file(filename)`` returns a file_handle - ``close_xlsx_file(file_handle)`` - - ``sheet_names list_sheets(file_handle)`` - - ``sheet_handle open_sheet(file_handle, sheet_name)`` + - ``list_sheets(file_handle)`` returns a list of strings representing sheet + names + - ``open_sheet(file_handle, sheet_name)`` returns a sheet_handle - ``close_sheet(sheet_handle)`` - - ``cell_strings get_row(sheet_handle, max_tokens)`` The ``max_tokens`` - parameter is optional. If set to a number > 0, it limits the number of cells - read and returned for the row. + - ``get_row(sheet_handle, max_tokens)`` returns a list of strings representing + the contents of the cells in the next row. The ``max_tokens`` parameter is + optional. If set to a number > 0, it limits the number of cells read and + returned for the row. - Lua users would be better off using the Lua class wrappers, though. For - example:: +The plugin also provides Lua class wrappers for ease of use: + +- ``XlsxioReader`` provides access to .xlsx files +- ``XlsxioSheetReader`` provides access to sheets within .xlsx files +- ``open(filepath)`` initializes and returns an ``XlsxioReader`` object + +The ``XlsxioReader`` class has the following methods: + +- ``XlsxioReader:close()`` closes the file. Be sure to close any open child + sheet handles first! +- ``XlsxioReader:list_sheets()`` returns a list of strings representing sheet + names +- ``XlsxioReader:open_sheet(sheet_name)`` returns an initialized + ``XlsxioSheetReader`` object + +The ``XlsxioSheetReader`` class has the following methods: + +- ``XlsxioSheetReader:close()`` closes the sheet +- ``XlsxioSheetReader:get_row(max_tokens)`` reads the next row from the sheet. + If ``max_tokens`` is specified and is a positive integer, only the first + ``max_tokens`` elements of the row are returned. + +Here is an end-to-end example:: local xlsxreader = require('plugins.xlsxreader') @@ -4337,7 +4360,7 @@ following low-level API methods: end local filepath = 'path/to/some_file.xlsx' - local reader = xlsxreader.XlsxioReader{filepath=filepath} + local reader = xlsxreader.open(filepath) dfhack.with_finalize( function() reader:close() end, function() diff --git a/plugins/lua/xlsxreader.lua b/plugins/lua/xlsxreader.lua index b9827c39f..f84c66d43 100644 --- a/plugins/lua/xlsxreader.lua +++ b/plugins/lua/xlsxreader.lua @@ -27,6 +27,10 @@ XlsxioReader.ATTRS{ filepath = DEFAULT_NIL, } +function open(filepath) + return XlsxioReader{filepath=filepath} +end + function XlsxioReader:init() if not self.filepath then error('XlsxReader: filepath is required') From 4ef0ff2a241b3d639120e1cf08a6e77aca3c0149 Mon Sep 17 00:00:00 2001 From: myk002 Date: Mon, 22 Mar 2021 10:40:29 -0700 Subject: [PATCH 4/6] fix formatting errors --- docs/Lua API.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/Lua API.rst b/docs/Lua API.rst index 83eef2956..261737157 100644 --- a/docs/Lua API.rst +++ b/docs/Lua API.rst @@ -4307,13 +4307,13 @@ xlsxreader Utility functions to facilitate reading .xlsx spreadsheets. It provides the following low-level API methods: - - ``open_xlsx_file(filename)`` returns a file_handle - - ``close_xlsx_file(file_handle)`` - - ``list_sheets(file_handle)`` returns a list of strings representing sheet +- ``open_xlsx_file(filename)`` returns an open file_handle +- ``close_xlsx_file(file_handle)`` +- ``list_sheets(file_handle)`` returns a list of strings representing sheet names - - ``open_sheet(file_handle, sheet_name)`` returns a sheet_handle - - ``close_sheet(sheet_handle)`` - - ``get_row(sheet_handle, max_tokens)`` returns a list of strings representing +- ``open_sheet(file_handle, sheet_name)`` returns a sheet_handle +- ``close_sheet(sheet_handle)`` +- ``get_row(sheet_handle, max_tokens)`` returns a list of strings representing the contents of the cells in the next row. The ``max_tokens`` parameter is optional. If set to a number > 0, it limits the number of cells read and returned for the row. From 222feff34251df5805735c5a063f0ec78c295a16 Mon Sep 17 00:00:00 2001 From: myk002 Date: Mon, 22 Mar 2021 10:44:32 -0700 Subject: [PATCH 5/6] expand docs for open and close functions --- docs/Lua API.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/Lua API.rst b/docs/Lua API.rst index 261737157..e6fbf038b 100644 --- a/docs/Lua API.rst +++ b/docs/Lua API.rst @@ -4307,12 +4307,14 @@ xlsxreader Utility functions to facilitate reading .xlsx spreadsheets. It provides the following low-level API methods: -- ``open_xlsx_file(filename)`` returns an open file_handle -- ``close_xlsx_file(file_handle)`` +- ``open_xlsx_file(filename)`` returns a file_handle or nil on error +- ``close_xlsx_file(file_handle)`` closes the specified file_handle - ``list_sheets(file_handle)`` returns a list of strings representing sheet names -- ``open_sheet(file_handle, sheet_name)`` returns a sheet_handle -- ``close_sheet(sheet_handle)`` +- ``open_sheet(file_handle, sheet_name)`` returns a sheet_handle. This call + always succeeds, even if the sheet doesn't exist. Non-existent sheets will + have no data, though. +- ``close_sheet(sheet_handle)`` closes the specified sheet_handle - ``get_row(sheet_handle, max_tokens)`` returns a list of strings representing the contents of the cells in the next row. The ``max_tokens`` parameter is optional. If set to a number > 0, it limits the number of cells read and From 4d2af8438f2fd3996c832553fd892f381e2145e6 Mon Sep 17 00:00:00 2001 From: myk002 Date: Fri, 26 Mar 2021 19:43:15 -0700 Subject: [PATCH 6/6] always read all cells to work around xlsxio bug otherwise xlsxio will return a spurious empty row on next row read --- plugins/xlsxreader.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/plugins/xlsxreader.cpp b/plugins/xlsxreader.cpp index d9a781f11..e17be096b 100644 --- a/plugins/xlsxreader.cpp +++ b/plugins/xlsxreader.cpp @@ -115,9 +115,12 @@ int get_row(lua_State *L) { std::string value; auto cells = std::vector(); while (get_next_cell(sheet_handle, value)) { - cells.push_back(value); - if (max_tokens > 0 && cells.size() >= max_tokens) - break; + // read all cells in the row, even if we don't need to; + // otherwise xlsxio will return a spurious empty row on + // next call + if (max_tokens <= 0 || cells.size() < max_tokens) { + cells.push_back(value); + } } Lua::PushVector(L, cells, true); }