Merge branch 'develop' into dig_digtype_nohidden

docs/dev/Lua API.rst

@@ -2594,11 +2594,18 @@ invalidates the ``texpos`` value that used to point to that texture.
 The ``textures`` module solves this problem by providing a stable handle instead of a
 raw ``texpos``. When we need to draw a particular tile, we can look up the current
 ``texpos`` value via the handle.
+Texture module can register textures in two ways: to reserved and dynamic ranges.
+Reserved range is a limit buffer in a game texture vector, that will never be wiped.
+It is good for static assets, which need to be loaded at the very beginning and will be used during the process running.
+In other cases, it is better to use dynamic range.
+If reserved range buffer limit has been reached, dynamic range will be used by default.
 
-* ``loadTileset(file, tile_px_w, tile_px_h)``
+* ``loadTileset(file, tile_px_w, tile_px_h[, reserved])``
 
   Loads a tileset from the image ``file`` with give tile dimensions in pixels. The
   image will be sliced in row major order. Returns an array of ``TexposHandle``.
+  ``reserved`` is optional boolean argument, which indicates texpos range.
+  ``true`` - reserved, ``false`` - dynamic (default).
 
   Example usage::
 
@@ -2611,18 +2618,22 @@ raw ``texpos``. When we need to draw a particular tile, we can look up the curre
   get the ``texpos`` for your texture. ``texpos`` can change when game textures are
   reset, but the handle will be the same.
 
-* ``createTile(pixels, tile_px_w, tile_px_h)``
+* ``createTile(pixels, tile_px_w, tile_px_h[, reserved])``
 
   Create and register a new texture with the given tile dimensions and an array of
   ``pixels`` in row major order. Each pixel is an integer representing color in packed
   RBGA format (for example, #0022FF11). Returns a ``TexposHandle``.
+  ``reserved`` is optional boolean argument, which indicates texpos range.
+  ``true`` - reserved, ``false`` - dynamic (default).
 
-* ``createTileset(pixels, texture_px_w, texture_px_h, tile_px_w, tile_px_h)``
+* ``createTileset(pixels, texture_px_w, texture_px_h, tile_px_w, tile_px_h[, reserved])``
 
   Create and register a new texture with the given texture dimensions and an array of
   ``pixels`` in row major order. Then slice it into tiles with the given tile
   dimensions. Each pixel is an integer representing color in packed RBGA format (for
   example #0022FF11). Returns an array of ``TexposHandle``.
+  ``reserved`` is optional boolean argument, which indicates texpos range.
+  ``true`` - reserved, ``false`` - dynamic (default).
 
 * ``deleteHandle(handle)``
 

library/LuaApi.cpp

@@ -1760,7 +1760,8 @@ static int textures_loadTileset(lua_State *state)
     std::string file = luaL_checkstring(state, 1);
     auto tile_w = luaL_checkint(state, 2);
     auto tile_h = luaL_checkint(state, 3);
-    auto handles = Textures::loadTileset(file, tile_w, tile_h);
+    bool reserved = lua_isboolean(state, 4) ? lua_toboolean(state, 4) : false;
+    auto handles = Textures::loadTileset(file, tile_w, tile_h, reserved);
     Lua::PushVector(state, handles);
     return 1;
 }
@@ -1798,7 +1799,8 @@ static int textures_createTile(lua_State *state)
     Lua::GetVector(state, pixels);
     auto tile_w = luaL_checkint(state, 2);
     auto tile_h = luaL_checkint(state, 3);
-    auto handle = Textures::createTile(pixels, tile_w, tile_h);
+    bool reserved = lua_isboolean(state, 4) ? lua_toboolean(state, 4) : false;
+    auto handle = Textures::createTile(pixels, tile_w, tile_h, reserved);
     Lua::Push(state, handle);
     return 1;
 }
@@ -1811,7 +1813,8 @@ static int textures_createTileset(lua_State *state)
     auto texture_h = luaL_checkint(state, 3);
     auto tile_w = luaL_checkint(state, 4);
     auto tile_h = luaL_checkint(state, 5);
-    auto handles = Textures::createTileset(pixels, texture_w, texture_h, tile_w, tile_h);
+    bool reserved = lua_isboolean(state, 6) ? lua_toboolean(state, 6) : false;
+    auto handles = Textures::createTileset(pixels, texture_w, texture_h, tile_w, tile_h, reserved);
     Lua::PushVector(state, handles);
     return 1;
 }

library/include/modules/Textures.h

@@ -26,7 +26,7 @@ const uint32_t TILE_HEIGHT_PX = 12;
  * Load texture and get handle.
  * Keep it to obtain valid texpos.
  */
-DFHACK_EXPORT TexposHandle loadTexture(SDL_Surface* surface);
+DFHACK_EXPORT TexposHandle loadTexture(SDL_Surface* surface, bool reserved = false);
 
 /**
  * Load tileset from image file.
@@ -34,7 +34,8 @@ DFHACK_EXPORT TexposHandle loadTexture(SDL_Surface* surface);
  */
 DFHACK_EXPORT std::vector<TexposHandle> loadTileset(const std::string& file,
                                                     int tile_px_w = TILE_WIDTH_PX,
-                                                    int tile_px_h = TILE_HEIGHT_PX);
+                                                    int tile_px_h = TILE_HEIGHT_PX,
+                                                    bool reserved = false);
 
 /**
  * Get texpos by handle.
@@ -53,7 +54,7 @@ DFHACK_EXPORT void deleteHandle(TexposHandle handle);
  * Register this texture and return TexposHandle.
  */
 DFHACK_EXPORT TexposHandle createTile(std::vector<uint32_t>& pixels, int tile_px_w = TILE_WIDTH_PX,
-                                      int tile_px_h = TILE_HEIGHT_PX);
+                                      int tile_px_h = TILE_HEIGHT_PX, bool reserved = false);
 
 /**
  * Create new textures as tileset with RGBA32 format and pixels as data in row major order.
@@ -62,7 +63,8 @@ DFHACK_EXPORT TexposHandle createTile(std::vector<uint32_t>& pixels, int tile_px
 DFHACK_EXPORT std::vector<TexposHandle> createTileset(std::vector<uint32_t>& pixels,
                                                       int texture_px_w, int texture_px_h,
                                                       int tile_px_w = TILE_WIDTH_PX,
-                                                      int tile_px_h = TILE_HEIGHT_PX);
+                                                      int tile_px_h = TILE_HEIGHT_PX,
+                                                      bool reserved = false);
 
 /**
  * Call this on DFHack init just once to setup interposed handlers and

library/lua/gui/textures.lua

@@ -6,18 +6,18 @@ local _ENV = mkmodule('gui.textures')
 
 -- Preloaded DFHack Assets.
 -- Use this handles if you need to get dfhack standard textures.
----@type table<string, TexposHandle>
+---@type table<string, TexposHandle[]>
 local texpos_handles = {
-  green_pin = dfhack.textures.loadTileset('hack/data/art/green-pin.png', 8, 12),
+  green_pin = dfhack.textures.loadTileset('hack/data/art/green-pin.png', 8, 12, true),
-  red_pin = dfhack.textures.loadTileset('hack/data/art/red-pin.png', 8, 12),
+  red_pin = dfhack.textures.loadTileset('hack/data/art/red-pin.png', 8, 12, true),
-  icons = dfhack.textures.loadTileset('hack/data/art/icons.png', 8, 12),
+  icons = dfhack.textures.loadTileset('hack/data/art/icons.png', 8, 12, true),
-  on_off = dfhack.textures.loadTileset('hack/data/art/on-off.png', 8, 12),
+  on_off = dfhack.textures.loadTileset('hack/data/art/on-off.png', 8, 12, true),
-  control_panel = dfhack.textures.loadTileset('hack/data/art/control-panel.png', 8, 12),
+  control_panel = dfhack.textures.loadTileset('hack/data/art/control-panel.png', 8, 12, true),
-  border_thin = dfhack.textures.loadTileset('hack/data/art/border-thin.png', 8, 12),
+  border_thin = dfhack.textures.loadTileset('hack/data/art/border-thin.png', 8, 12, true),
-  border_medium = dfhack.textures.loadTileset('hack/data/art/border-medium.png', 8, 12),
+  border_medium = dfhack.textures.loadTileset('hack/data/art/border-medium.png', 8, 12, true),
-  border_bold = dfhack.textures.loadTileset('hack/data/art/border-bold.png', 8, 12),
+  border_bold = dfhack.textures.loadTileset('hack/data/art/border-bold.png', 8, 12, true),
-  border_panel = dfhack.textures.loadTileset('hack/data/art/border-panel.png', 8, 12),
+  border_panel = dfhack.textures.loadTileset('hack/data/art/border-panel.png', 8, 12, true),
-  border_window = dfhack.textures.loadTileset('hack/data/art/border-window.png', 8, 12),
+  border_window = dfhack.textures.loadTileset('hack/data/art/border-window.png', 8, 12, true),
 }
 
 -- Get valid texpos for preloaded texture in tileset

library/modules/Textures.cpp

@@ -1,3 +1,5 @@
+#include <algorithm>
+#include <atomic>
 #include <mutex>
 #include <numeric>
 #include <unordered_map>
@@ -28,9 +30,35 @@ namespace DFHack {
 DBG_DECLARE(core, textures, DebugCategory::LINFO);
 }
 
+struct ReservedRange {
+    void init(int32_t start) {
+        this->start = start;
+        this->end = start + ReservedRange::size;
+        this->current = start;
+        this->is_installed = true;
+    }
+    long get_new_texpos() {
+        if (this->current == this->end)
+            return -1;
+        return this->current++;
+    }
+
+    static const int32_t size = 10000; // size of reserved texpos buffer
+    int32_t start = -1;
+    int32_t end = -1;
+    long current = -1;
+    bool is_installed = false;
+};
+
+static ReservedRange reserved_range{};
 static std::unordered_map<TexposHandle, long> g_handle_to_texpos;
+static std::unordered_map<TexposHandle, long> g_handle_to_reserved_texpos;
 static std::unordered_map<TexposHandle, SDL_Surface*> g_handle_to_surface;
+static std::unordered_map<std::string, std::vector<TexposHandle>> g_tileset_to_handles;
+static std::vector<TexposHandle> g_delayed_regs;
 static std::mutex g_adding_mutex;
+static std::atomic<bool> loading_state = false;
+static SDL_Surface* dummy_surface = NULL;
 
 // Converts an arbitrary Surface to something like the display format
 // (32-bit RGBA), and converts magenta to transparency if convert_magenta is set
@@ -71,6 +99,12 @@ static long add_texture(SDL_Surface* surface) {
     return texpos;
 }
 
+// register surface in texture raws to specific texpos
+static void insert_texture(SDL_Surface* surface, long texpos) {
+    std::lock_guard<std::mutex> lg_add_texture(g_adding_mutex);
+    enabler->textures.raws[texpos] = surface;
+}
+
 // delete surface from texture raws
 static void delete_texture(long texpos) {
     std::lock_guard<std::mutex> lg_add_texture(g_adding_mutex);
@@ -94,7 +128,8 @@ SDL_Surface* create_texture(std::vector<uint32_t>& pixels, int texture_px_w, int
 
 // convert single surface into tiles according w/h
 // register tiles in texture raws and return handles
-std::vector<TexposHandle> slice_tileset(SDL_Surface* surface, int tile_px_w, int tile_px_h) {
+std::vector<TexposHandle> slice_tileset(SDL_Surface* surface, int tile_px_w, int tile_px_h,
+                                        bool reserved) {
     std::vector<TexposHandle> handles{};
     if (!surface)
         return handles;
@@ -102,6 +137,12 @@ std::vector<TexposHandle> slice_tileset(SDL_Surface* surface, int tile_px_w, int
     int dimx = surface->w / tile_px_w;
     int dimy = surface->h / tile_px_h;
 
+    if (reserved && (dimx * dimy > reserved_range.end - reserved_range.current)) {
+        WARN(textures).print(
+            "there is not enough space in reserved range for whole tileset, using dynamic range\n");
+        reserved = false;
+    }
+
     for (int y = 0; y < dimy; y++) {
         for (int x = 0; x < dimx; x++) {
             SDL_Surface* tile = DFSDL_CreateRGBSurface(
@@ -109,7 +150,7 @@ std::vector<TexposHandle> slice_tileset(SDL_Surface* surface, int tile_px_w, int
                 surface->format->Bmask, surface->format->Amask);
             SDL_Rect vp{tile_px_w * x, tile_px_h * y, tile_px_w, tile_px_h};
             DFSDL_UpperBlit(surface, &vp, tile, NULL);
-            auto handle = Textures::loadTexture(tile);
+            auto handle = Textures::loadTexture(tile, reserved);
             handles.push_back(handle);
         }
     }
@@ -118,20 +159,46 @@ std::vector<TexposHandle> slice_tileset(SDL_Surface* surface, int tile_px_w, int
     return handles;
 }
 
-TexposHandle Textures::loadTexture(SDL_Surface* surface) {
+TexposHandle Textures::loadTexture(SDL_Surface* surface, bool reserved) {
     if (!surface || !enabler)
         return 0; // should be some error, i guess
+    if (loading_state)
+        reserved = true; // use reserved range during loading for all textures
 
     auto handle = reinterpret_cast<uintptr_t>(surface);
     g_handle_to_surface.emplace(handle, surface);
     surface->refcount++; // prevent destruct on next FreeSurface by game
-    auto texpos = add_texture(surface);
+
-    g_handle_to_texpos.emplace(handle, texpos);
+    if (reserved && reserved_range.is_installed) {
+        auto texpos = reserved_range.get_new_texpos();
+        if (texpos != -1) {
+            insert_texture(surface, texpos);
+            g_handle_to_reserved_texpos.emplace(handle, texpos);
+            dummy_surface->refcount--;
+            return handle;
+        }
+
+        if (loading_state) { // if we in loading state and reserved range is full -> error
+            ERR(textures).printerr("reserved range limit has been reached, use dynamic range\n");
+            return 0;
+        }
+    }
+
+    // if we here in loading state = true, then it should be dynamic range -> delay reg
+    if (loading_state) {
+        g_delayed_regs.push_back(handle);
+    } else {
+        auto texpos = add_texture(surface);
+        g_handle_to_texpos.emplace(handle, texpos);
+    }
+
     return handle;
 }
 
 std::vector<TexposHandle> Textures::loadTileset(const std::string& file, int tile_px_w,
-                                                int tile_px_h) {
+                                                int tile_px_h, bool reserved) {
+    if (g_tileset_to_handles.contains(file))
+        return g_tileset_to_handles[file];
     if (!enabler)
         return std::vector<TexposHandle>{};
 
@@ -142,9 +209,10 @@ std::vector<TexposHandle> Textures::loadTileset(const std::string& file, int til
     }
 
     surface = canonicalize_format(surface);
-    auto handles = slice_tileset(surface, tile_px_w, tile_px_h);
+    auto handles = slice_tileset(surface, tile_px_w, tile_px_h, reserved);
 
     DEBUG(textures).print("loaded %zd textures from '%s'\n", handles.size(), file.c_str());
+    g_tileset_to_handles[file] = handles;
 
     return handles;
 }
@@ -153,11 +221,18 @@ long Textures::getTexposByHandle(TexposHandle handle) {
     if (!handle || !enabler)
         return -1;
 
+    if (g_handle_to_reserved_texpos.contains(handle))
+        return g_handle_to_reserved_texpos[handle];
     if (g_handle_to_texpos.contains(handle))
         return g_handle_to_texpos[handle];
-
+    if (std::find(g_delayed_regs.begin(), g_delayed_regs.end(), handle) != g_delayed_regs.end())
+        return 0;
     if (g_handle_to_surface.contains(handle)) {
         g_handle_to_surface[handle]->refcount++; // prevent destruct on next FreeSurface by game
+        if (loading_state) { // reinit dor dynamic range during loading -> delayed
+            g_delayed_regs.push_back(handle);
+            return 0;
+        }
         auto texpos = add_texture(g_handle_to_surface[handle]);
         g_handle_to_texpos.emplace(handle, texpos);
         return texpos;
@@ -166,22 +241,24 @@ long Textures::getTexposByHandle(TexposHandle handle) {
     return -1;
 }
 
-TexposHandle Textures::createTile(std::vector<uint32_t>& pixels, int tile_px_w, int tile_px_h) {
+TexposHandle Textures::createTile(std::vector<uint32_t>& pixels, int tile_px_w, int tile_px_h,
+                                  bool reserved) {
     if (!enabler)
         return 0;
 
     auto texture = create_texture(pixels, tile_px_w, tile_px_h);
-    auto handle = Textures::loadTexture(texture);
+    auto handle = Textures::loadTexture(texture, reserved);
     return handle;
 }
 
 std::vector<TexposHandle> Textures::createTileset(std::vector<uint32_t>& pixels, int texture_px_w,
-                                                  int texture_px_h, int tile_px_w, int tile_px_h) {
+                                                  int texture_px_h, int tile_px_w, int tile_px_h,
+                                                  bool reserved) {
     if (!enabler)
         return std::vector<TexposHandle>{};
 
     auto texture = create_texture(pixels, texture_px_w, texture_px_h);
-    auto handles = slice_tileset(texture, tile_px_w, tile_px_h);
+    auto handles = slice_tileset(texture, tile_px_w, tile_px_h, reserved);
     return handles;
 }
 
@@ -192,8 +269,13 @@ void Textures::deleteHandle(TexposHandle handle) {
     auto texpos = Textures::getTexposByHandle(handle);
     if (texpos > 0)
         delete_texture(texpos);
+    if (g_handle_to_reserved_texpos.contains(handle))
+        g_handle_to_reserved_texpos.erase(handle);
     if (g_handle_to_texpos.contains(handle))
         g_handle_to_texpos.erase(handle);
+    if (auto it = std::find(g_delayed_regs.begin(), g_delayed_regs.end(), handle);
+        it != g_delayed_regs.end())
+        g_delayed_regs.erase(it);
     if (g_handle_to_surface.contains(handle)) {
         auto surface = g_handle_to_surface[handle];
         while (surface->refcount)
@@ -207,20 +289,45 @@ static void reset_texpos() {
     g_handle_to_texpos.clear();
 }
 
+static void reset_reserved_texpos() {
+    DEBUG(textures).print("resetting reserved texture mappings\n");
+    g_handle_to_reserved_texpos.clear();
+}
+
+static void reset_tilesets() {
+    DEBUG(textures).print("resetting tileset to handle mappings\n");
+    g_tileset_to_handles.clear();
+}
+
 static void reset_surface() {
+    DEBUG(textures).print("deleting cached surfaces\n");
     for (auto& entry : g_handle_to_surface) {
         DFSDL_FreeSurface(entry.second);
     }
     g_handle_to_surface.clear();
 }
 
+static void register_delayed_handles() {
+    DEBUG(textures).print("register delayed handles, size %zd\n", g_delayed_regs.size());
+    for (auto& handle : g_delayed_regs) {
+        auto texpos = add_texture(g_handle_to_surface[handle]);
+        g_handle_to_texpos.emplace(handle, texpos);
+    }
+    g_delayed_regs.clear();
+}
+
 // reset point on New Game
 struct tracking_stage_new_region : df::viewscreen_new_regionst {
     typedef df::viewscreen_new_regionst interpose_base;
 
     DEFINE_VMETHOD_INTERPOSE(void, logic, ()) {
         if (this->m_raw_load_stage != this->raw_load_stage) {
-            TRACE(textures).print("raw_load_stage %d -> %d\n", this->m_raw_load_stage, this->raw_load_stage);
+            TRACE(textures).print("raw_load_stage %d -> %d\n", this->m_raw_load_stage,
+                                  this->raw_load_stage);
+            bool tmp_state = loading_state;
+            loading_state = this->raw_load_stage >= 0 && this->raw_load_stage < 3 ? true : false;
+            if (tmp_state != loading_state && !loading_state)
+                register_delayed_handles();
             this->m_raw_load_stage = this->raw_load_stage;
             if (this->m_raw_load_stage == 1)
                 reset_texpos();
@@ -240,6 +347,10 @@ struct tracking_stage_adopt_region : df::viewscreen_adopt_regionst {
     DEFINE_VMETHOD_INTERPOSE(void, logic, ()) {
         if (this->m_cur_step != this->cur_step) {
             TRACE(textures).print("step %d -> %d\n", this->m_cur_step, this->cur_step);
+            bool tmp_state = loading_state;
+            loading_state = this->cur_step >= 0 && this->cur_step < 3 ? true : false;
+            if (tmp_state != loading_state && !loading_state)
+                register_delayed_handles();
             this->m_cur_step = this->cur_step;
             if (this->m_cur_step == 1)
                 reset_texpos();
@@ -259,6 +370,10 @@ struct tracking_stage_load_region : df::viewscreen_loadgamest {
     DEFINE_VMETHOD_INTERPOSE(void, logic, ()) {
         if (this->m_cur_step != this->cur_step) {
             TRACE(textures).print("step %d -> %d\n", this->m_cur_step, this->cur_step);
+            bool tmp_state = loading_state;
+            loading_state = this->cur_step >= 0 && this->cur_step < 3 ? true : false;
+            if (tmp_state != loading_state && !loading_state)
+                register_delayed_handles();
             this->m_cur_step = this->cur_step;
             if (this->m_cur_step == 1)
                 reset_texpos();
@@ -278,6 +393,10 @@ struct tracking_stage_new_arena : df::viewscreen_new_arenast {
     DEFINE_VMETHOD_INTERPOSE(void, logic, ()) {
         if (this->m_cur_step != this->cur_step) {
             TRACE(textures).print("step %d -> %d\n", this->m_cur_step, this->cur_step);
+            bool tmp_state = loading_state;
+            loading_state = this->cur_step >= 0 && this->cur_step < 3 ? true : false;
+            if (tmp_state != loading_state && !loading_state)
+                register_delayed_handles();
             this->m_cur_step = this->cur_step;
             if (this->m_cur_step == 0)
                 reset_texpos();
@@ -304,12 +423,31 @@ static void uninstall_reset_point() {
     INTERPOSE_HOOK(tracking_stage_new_arena, logic).remove();
 }
 
+static void reserve_static_range() {
+    if (static_cast<size_t>(enabler->textures.init_texture_size) != enabler->textures.raws.size()) {
+        WARN(textures).print(
+            "reserved range can't be installed! all textures will be loaded to dynamic range!");
+        return;
+    }
+    reserved_range.init(enabler->textures.init_texture_size);
+    dummy_surface =
+        DFSDL_CreateRGBSurfaceWithFormat(0, 0, 0, 32, SDL_PixelFormatEnum::SDL_PIXELFORMAT_RGBA32);
+    dummy_surface->refcount += ReservedRange::size;
+    for (int32_t i = 0; i < ReservedRange::size; i++) {
+        add_texture(dummy_surface);
+    }
+    enabler->textures.init_texture_size += ReservedRange::size;
+}
+
 void Textures::init(color_ostream& out) {
     if (!enabler)
         return;
 
+    reserve_static_range();
     install_reset_point();
-    DEBUG(textures, out).print("dynamic texture loading ready");
+    DEBUG(textures, out)
+        .print("dynamic texture loading ready, reserved range %d-%d\n", reserved_range.start,
+               reserved_range.end);
 }
 
 void Textures::cleanup() {
@@ -317,6 +455,8 @@ void Textures::cleanup() {
         return;
 
     reset_texpos();
+    reset_reserved_texpos();
+    reset_tilesets();
     reset_surface();
     uninstall_reset_point();
 }

plugins/lua/hotkeys.lua

@@ -5,8 +5,8 @@ local helpdb = require('helpdb')
 local overlay = require('plugins.overlay')
 local widgets = require('gui.widgets')
 
-local logo_textures = dfhack.textures.loadTileset('hack/data/art/logo.png', 8, 12)
+local logo_textures = dfhack.textures.loadTileset('hack/data/art/logo.png', 8, 12, true)
-local logo_hovered_textures = dfhack.textures.loadTileset('hack/data/art/logo_hovered.png', 8, 12)
+local logo_hovered_textures = dfhack.textures.loadTileset('hack/data/art/logo_hovered.png', 8, 12, true)
 
 local function get_command(cmdline)
     local first_word = cmdline:trim():split(' +')[1]

plugins/pathable.cpp

@@ -36,7 +36,7 @@ namespace DFHack {
 static std::vector<TexposHandle> textures;
 
 DFhackCExport command_result plugin_init(color_ostream &out, std::vector<PluginCommand> &commands) {
-    textures = Textures::loadTileset("hack/data/art/pathable.png", 32, 32);
+    textures = Textures::loadTileset("hack/data/art/pathable.png", 32, 32, true);
     return CR_OK;
 }