diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 000000000..2367db678 --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,6 @@ +Thank you for your interest in contributing to DFHack! If you're reading this +document, you're probably viewing it on GitHub. The DFHack docs are hosted +on [ReadTheDocs](https://dfhack.readthedocs.io/) - in particular, contributing +guidelines are [here](https://docs.dfhack.org/en/latest/docs/Contributing.html). +Double-checking the style guidelines before submitting a pull request is +always appreciated. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 17af62cb1..fb98bfa0b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,6 +6,10 @@ jobs: build: runs-on: ubuntu-18.04 steps: + - name: Set up Python 3 + uses: actions/setup-python@v2 + with: + python-version: 3 - name: Install dependencies run: | sudo apt-get update @@ -18,18 +22,26 @@ jobs: lua5.3 \ ninja-build \ zlib1g-dev - sudo pip3 install --system sphinx + pip install sphinx - name: Clone DFHack - uses: actions/checkout@v1 + uses: actions/checkout@v2 with: + fetch-depth: 0 # unlimited - we need past tags submodules: true - name: Set up environment + id: env_setup run: | - echo export DF_VERSION="$(sh travis/get-df-version.sh)" >> "$HOME/.df-env" - echo export DF_FOLDER="$HOME/DF/$DF_VERSION/df_linux" >> "$HOME/.df-env" + DF_VERSION="$(sh travis/get-df-version.sh)" + echo "::set-env name=DF_VERSION::${DF_VERSION}" + echo "::set-output name=df_version::${DF_VERSION}" + echo "::set-env name=DF_FOLDER::${HOME}/DF/${DF_VERSION}/df_linux" + - name: Fetch DF cache + uses: actions/cache@v2 + with: + path: ~/DF + key: ${{ steps.env_setup.outputs.df_version }} - name: Download DF run: | - source "$HOME/.df-env" sh travis/download-df.sh - name: Build docs run: | @@ -41,7 +53,6 @@ jobs: path: docs/html - name: Build DFHack run: | - source "$HOME/.df-env" cmake \ -S . \ -B build-ci \ @@ -53,7 +64,6 @@ jobs: ninja -C build-ci install - name: Run tests run: | - source "$HOME/.df-env" export TERM=dumb mv "$DF_FOLDER"/dfhack.init-example "$DF_FOLDER"/dfhack.init script -qe -c "python travis/run-tests.py --headless --keep-status \"$DF_FOLDER\"" @@ -66,34 +76,53 @@ jobs: with: name: test-artifacts path: artifacts + - name: Clean up DF folder + # to prevent DFHack-generated files from ending up in the cache + # (download-df.sh also removes them, this is just to save cache space) + if: success() || failure() + run: | + rm -rf "$DF_FOLDER" + lint: runs-on: ubuntu-18.04 steps: - - name: Install dependencies + - name: Set up Python 3 + uses: actions/setup-python@v2 + with: + python-version: 3 + - name: Set up Ruby 2.7 + uses: actions/setup-ruby@v1 + with: + ruby-version: 2.7 + - name: Install Lua run: | sudo apt-get update - sudo apt-get install \ - lua5.3 \ - ruby + sudo apt-get install lua5.3 - name: Clone DFHack - uses: actions/checkout@v1 + uses: actions/checkout@v2 with: submodules: true + # don't need tags here - name: Check whitespace run: | python travis/lint.py - name: Check Authors.rst + if: success() || failure() run: | python travis/authors-rst.py - name: Check for missing documentation + if: success() || failure() run: | python travis/script-docs.py - name: Check Lua syntax + if: success() || failure() run: | - python travis/script-syntax.py --ext=lua --cmd="luac5.3 -p" + python travis/script-syntax.py --ext=lua --cmd="luac5.3 -p" --github-actions - name: Check Ruby syntax + if: success() || failure() run: | - python travis/script-syntax.py --ext=rb --cmd="ruby -c" + python travis/script-syntax.py --ext=rb --cmd="ruby -c" --github-actions + check-pr: runs-on: ubuntu-latest if: github.event_name == 'pull_request' diff --git a/CMakeLists.txt b/CMakeLists.txt index 5a5e8312b..de0c858af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -89,6 +89,10 @@ if(MSVC) # a smaller type, and most of the time this is just conversion from 64 to 32 bits # for things like vector sizes, which are never that big anyway. add_definitions("/wd4267") + + # MSVC panics if an object file contains more than 65,279 sections. this + # happens quite frequently with code that uses templates, such as vectors. + add_definitions("/bigobj") endif() # Automatically detect architecture based on Visual Studio generator diff --git a/Contributing.rst b/Contributing.rst deleted file mode 100644 index e358e66b3..000000000 --- a/Contributing.rst +++ /dev/null @@ -1,229 +0,0 @@ -########################### -How to contribute to DFHack -########################### - -.. contents:: - - -.. _contributing-code: - -Contributing Code -================= -Several things should be kept in mind when contributing code to DFHack. - -Code Format ------------ -* Four space indents for C++. Never use tabs for indentation in any language. -* LF (Unix style) line terminators -* Avoid trailing whitespace -* UTF-8 encoding -* For C++: - - * Opening and closing braces on their own lines or opening brace at the end of the previous line - * Braces placed at original indent level if on their own lines - * #includes should be sorted. C++ libraries first, then dfhack modules, then df structures, - then local includes. Within each category they should be sorted alphabetically. - -How to get new code into DFHack -------------------------------- -* Submit pull requests to the ``develop`` branch, not the ``master`` branch. - (The ``master`` branch always points at the most recent release) -* Use a new branch for each feature or bugfix so that your changes can be merged independently - (i.e. not the master or develop branch of your fork). -* If possible, compile on multiple platforms when changing anything that compiles -* It must pass CI - run ``python travis/all.py`` to check this. -* Update ``changelog.txt`` and ``docs/Authors.rst`` when applicable. See - `build-changelog` for more information on the changelog format. -* Create a GitHub pull request once finished -* Submit ideas and bug reports as :issue:`issues on GitHub <>`. - Posts in the forum thread can easily get missed or forgotten. -* Work on :issue:`reported problems ` - will take priority over ideas or suggestions. - -.. _contributing-memory-research: - -Memory research ---------------- -If you want to do memory research, you'll need some tools and some knowledge. -In general, you'll need a good memory viewer and optionally something -to look at machine code without getting crazy :) -Using publicly known information and analyzing the game's data is preferred. - -Good Windows tools include: - -* IDA Freeware 7.0 (for non-commercial use, supports 32-bit and 64-bit) -* Cheat Engine - -Good Linux tools: - -* angavrilov's df-structures gui (32-bit only, visit us on IRC for details) -* IDA Freeware 7.0 (see above) -* edb (Evan's Debugger) -* Some of the tools residing in the ``legacy`` dfhack branch. - -Using the library as a developer -================================ -Currently, the most direct way to use the library is to write a script or plugin that can be loaded by it. -All the plugins can be found in the 'plugins' folder. There's no in-depth documentation -on how to write one yet, but it should be easy enough to copy one and just follow the pattern. -``plugins/skeleton/skeleton.cpp`` is provided for this purpose. - -Other than through plugins, it is possible to use DFHack via remote access interface, -or by writing scripts in Lua or Ruby. There are plenty of examples in the scripts folder. -The `lua-api` is quite well documented. - -The most important parts of DFHack are the Core, Console, Modules and Plugins. - -* Core acts as the centerpiece of DFHack - it acts as a filter between DF and - SDL and synchronizes the various plugins with DF. -* Console is a thread-safe console that can be used to invoke commands exported by Plugins. -* Modules actually describe the way to access information in DF's memory. You - can get them from the Core. Most modules are split into two parts: high-level - and low-level. High-level is mostly method calls, low-level publicly visible - pointers to DF's data structures. -* Plugins are the tools that use all the other stuff to make things happen. - A plugin can have a list of commands that it exports and an onupdate function - that will be called each DF game tick. - -Rudimentary API documentation can be built using doxygen (see build options -in ``CMakeCache.txt`` or with ``ccmake`` or ``cmake-gui``). The full DFHack -documentation is built with Sphinx_, which runs automatically at compile time. - -.. _Sphinx: http://www.sphinx-doc.org - -DFHack consists of variously licensed code, but invariably weak copyleft. -The main license is zlib/libpng, some bits are MIT licensed, and some are -BSD licensed. See the `license` for more information. - -Feel free to add your own extensions and plugins. Contributing back to -the DFHack repository is welcome and the right thing to do :) - -DF data structure definitions ------------------------------ -DFHack uses information about the game data structures, represented via xml files -in the ``library/xml/`` submodule. - -See https://github.com/DFHack/df-structures, and the documentation linked in the index. - -Data structure layouts are described in files following the ``df.*.xml`` name pattern. -This information is transformed by a perl script into C++ headers describing the -structures, and associated metadata for the Lua wrapper. These headers and data -are then compiled into the DFHack libraries, thus necessitating a compatibility -break every time layouts change; in return it significantly boosts the efficiency -and capabilities of DFHack code. - -Global object addresses are stored in :file:`symbols.xml`, which is copied to the dfhack -release package and loaded as data at runtime. - -Remote access interface ------------------------ -DFHack supports remote access by exchanging Google protobuf messages via a TCP -socket. Both the core and plugins can define remotely accessible methods. The -``dfhack-run`` command uses this interface to invoke ordinary console commands. - -Currently the supported set of requests is limited, because the developers don't -know what exactly is most useful. `remotefortressreader` provides a fairly -comprehensive interface for visualisers such as :forums:`Armok Vision <146473>`. - - -Documentation Standards -======================= -DFHack documentation is built with Sphinx_, and configured automatically -through CMake. If you want to build the docs *only*, use this command:: - - sphinx-build . docs/html - -Whether you're adding new code or just fixing old documentation (and there's plenty), -there are a few important standards for completeness and consistent style. Treat -this section as a guide rather than iron law, match the surrounding text, and you'll -be fine. - -Each command should have a short (~54 character) help string, which is shown -by the `ls` command. For scripts, this is a comment on the first line -(the comment marker and whitespace is stripped). For plugins it's the second -argument to ``PluginCommand``. Please make this brief but descriptive! - -Everything should be documented! If it's not clear *where* a particular -thing should be documented, ask on IRC or in the DFHack thread on Bay12 - -as well as getting help, you'll be providing valuable feedback that -makes it easier for future readers! - -Scripts can use a custom autodoc function, based on the Sphinx ``include`` -directive - anything between the tokens is copied into the appropriate scripts -documentation page. For Ruby, we follow the built-in docstring convention -(``=begin`` and ``=end``). For Lua, the tokens are ``[====[`` and ``]====]`` -- ordinary multi-line strings. It is highly encouraged to reuse this string -as the in-console documentation by (eg.) printing it when a ``-help`` argument -is given. - -The docs **must** have a heading which exactly matches the command, underlined -with ``=====`` to the same length. For example, a lua file would have:: - - local helpstr = [====[ - - add-thought - =========== - Adds a thought or emotion to the selected unit. Can be used by other scripts, - or the gui invoked by running ``add-thought gui`` with a unit selected. - - ]====] - - -Where the heading for a section is also the name of a command, the spelling -and case should exactly match the command to enter in the DFHack command line. - -Try to keep lines within 80-100 characters, so it's readable in plain text -in the terminal - Sphinx (our documentation system) will make sure -paragraphs flow. - -If there aren't many options or examples to show, they can go in a paragraph of -text. Use double-backticks to put commands in monospaced font, like this:: - - You can use ``cleanowned scattered x`` to dump tattered or abandoned items. - -If the command takes more than three arguments, format the list as a table -called Usage. The table *only* lists arguments, not full commands. -Input values are specified in angle brackets. Example:: - - Usage: - - :arg1: A simple argument. - :arg2 : Does something based on the input value. - :Very long argument: - Is very specific. - -To demonstrate usage - useful mainly when the syntax is complicated, list the -full command with arguments in monospaced font, then indent the next line and -describe the effect:: - - ``resume all`` - Resumes all suspended constructions. - -If it would be helpful to mention another DFHack command, don't just type the -name - add a hyperlink! Specify the link target in backticks, and it will be -replaced with the corresponding title and linked: eg ```autolabor``` -=> `autolabor`. Link targets should be equivalent to the command -described (without file extension), and placed above the heading of that -section like this:: - - .. _autolabor: - - autolabor - ========= - -Add link targets if you need them, but otherwise plain headings are preferred. -Scripts have link targets created automatically. - -Other ways to help -================== -DFHack is a software project, but there's a lot more to it than programming. -If you're not comfortable programming, you can help by: - -* reporting bugs and incomplete documentation -* improving the documentation -* finding third-party scripts to add -* writing tutorials for newbies - -All those things are crucial, and often under-represented. So if that's -your thing, go get started! - diff --git a/conf.py b/conf.py index 2f47d5644..a46039126 100644 --- a/conf.py +++ b/conf.py @@ -15,14 +15,13 @@ serve to show the default. # pylint:disable=redefined-builtin +import datetime from io import open import os import re import shlex # pylint:disable=unused-import import sys -sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'docs')) -from gen_changelog import generate_changelog # -- Support :dfhack-keybind:`command` ------------------------------------ # this is a custom directive that pulls info from dfhack.init-example @@ -152,9 +151,10 @@ def write_script_docs(): 'gui': 'GUI Scripts', 'modtools': 'Scripts for Modders'} for k in head: - title = ('.. _{k}:\n\n{l}\n{t}\n{l}\n\n' + title = ('.. _scripts-{k}:\n\n{l}\n{t}\n{l}\n\n' '.. include:: /scripts/{a}about.txt\n\n' - '.. contents::\n\n').format( + '.. contents:: Contents\n' + ' :local:\n\n').format( k=k, t=head[k], l=len(head[k])*'#', a=('' if k == 'base' else k + '/') @@ -179,19 +179,23 @@ def all_keybinds_documented(): # Actually call the docs generator and run test -generate_changelog() write_script_docs() all_keybinds_documented() # -- General configuration ------------------------------------------------ +sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'docs', 'sphinx_extensions')) + # If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = '1.3' +needs_sphinx = '1.8' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.extlinks'] +extensions = [ + 'sphinx.ext.extlinks', + 'dfhack.changelog', +] # This config value must be a dictionary of external sites, mapping unique # short alias names to a base URL and a prefix. @@ -208,7 +212,7 @@ extlinks = { } # Add any paths that contain templates here, relative to this directory. -templates_path = [] +templates_path = ["docs/templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: @@ -222,7 +226,7 @@ master_doc = 'index' # General information about the project. project = 'DFHack' -copyright = '2015, The DFHack Team' +copyright = '2015-%d, The DFHack Team' % datetime.datetime.now().year author = 'The DFHack Team' # The version info for the project you're documenting, acts as replacement for @@ -279,13 +283,19 @@ pygments_style = 'sphinx' # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False +rst_prolog = """ +.. |sphinx_min_version| replace:: {sphinx_min_version} +.. |dfhack_version| replace:: {dfhack_version} +""".format( + sphinx_min_version=needs_sphinx, + dfhack_version=version, +) # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'alabaster' -html_style = 'dfhack.css' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -296,6 +306,7 @@ html_theme_options = { 'github_repo': 'dfhack', 'github_button': False, 'travis_button': False, + 'fixed_sidebar': True, } # The name for this set of Sphinx documents. If None, it defaults to @@ -335,6 +346,10 @@ html_domain_indices = False # If false, no index is generated. html_use_index = False +html_css_files = [ + 'dfhack.css', +] + # -- Options for LaTeX output --------------------------------------------- # Grouping the document tree into LaTeX files. List of tuples diff --git a/depends/tthread/CMakeLists.txt b/depends/tthread/CMakeLists.txt index 9e0e85dc5..dfb1d9901 100644 --- a/depends/tthread/CMakeLists.txt +++ b/depends/tthread/CMakeLists.txt @@ -1,5 +1,5 @@ project(dfhack-tinythread) -add_library(dfhack-tinythread STATIC EXCLUDE_FROM_ALL tinythread.cpp tinythread.h fast_mutex.h) +add_library(dfhack-tinythread STATIC EXCLUDE_FROM_ALL tinythread.cpp tinythread.h) if(UNIX) target_link_libraries(dfhack-tinythread pthread) endif() diff --git a/depends/tthread/fast_mutex.h b/depends/tthread/fast_mutex.h deleted file mode 100644 index a381b2c75..000000000 --- a/depends/tthread/fast_mutex.h +++ /dev/null @@ -1,249 +0,0 @@ -/* -*- mode: c++; tab-width: 2; indent-tabs-mode: nil; -*- -Copyright (c) 2010-2012 Marcus Geelnard - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source - distribution. -*/ - -#ifndef _FAST_MUTEX_H_ -#define _FAST_MUTEX_H_ - -/// @file - -// Which platform are we on? -#if !defined(_TTHREAD_PLATFORM_DEFINED_) - #if defined(_WIN32) || defined(__WIN32__) || defined(__WINDOWS__) - #define _TTHREAD_WIN32_ - #else - #define _TTHREAD_POSIX_ - #endif - #define _TTHREAD_PLATFORM_DEFINED_ -#endif - -// Check if we can support the assembly language level implementation (otherwise -// revert to the system API) -#if (defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))) || \ - (defined(_MSC_VER) && (defined(_M_IX86) /*|| defined(_M_X64)*/)) || \ - (defined(__GNUC__) && (defined(__ppc__))) - #define _FAST_MUTEX_ASM_ -#else - #define _FAST_MUTEX_SYS_ -#endif - -#if defined(_TTHREAD_WIN32_) - #define NOMINMAX - #ifndef WIN32_LEAN_AND_MEAN - #define WIN32_LEAN_AND_MEAN - #define __UNDEF_LEAN_AND_MEAN - #endif - #include - #ifdef __UNDEF_LEAN_AND_MEAN - #undef WIN32_LEAN_AND_MEAN - #undef __UNDEF_LEAN_AND_MEAN - #endif -#else - #ifdef _FAST_MUTEX_ASM_ - #include - #else - #include - #endif -#endif - -namespace tthread { - -/// Fast mutex class. -/// This is a mutual exclusion object for synchronizing access to shared -/// memory areas for several threads. It is similar to the tthread::mutex class, -/// but instead of using system level functions, it is implemented as an atomic -/// spin lock with very low CPU overhead. -/// -/// The \c fast_mutex class is NOT compatible with the \c condition_variable -/// class (however, it IS compatible with the \c lock_guard class). It should -/// also be noted that the \c fast_mutex class typically does not provide -/// as accurate thread scheduling as a the standard \c mutex class does. -/// -/// Because of the limitations of the class, it should only be used in -/// situations where the mutex needs to be locked/unlocked very frequently. -/// -/// @note The "fast" version of this class relies on inline assembler language, -/// which is currently only supported for 32/64-bit Intel x86/AMD64 and -/// PowerPC architectures on a limited number of compilers (GNU g++ and MS -/// Visual C++). -/// For other architectures/compilers, system functions are used instead. -class fast_mutex { - public: - /// Constructor. -#if defined(_FAST_MUTEX_ASM_) - fast_mutex() : mLock(0) {} -#else - fast_mutex() - { - #if defined(_TTHREAD_WIN32_) - InitializeCriticalSection(&mHandle); - #elif defined(_TTHREAD_POSIX_) - pthread_mutex_init(&mHandle, NULL); - #endif - } -#endif - -#if !defined(_FAST_MUTEX_ASM_) - /// Destructor. - ~fast_mutex() - { - #if defined(_TTHREAD_WIN32_) - DeleteCriticalSection(&mHandle); - #elif defined(_TTHREAD_POSIX_) - pthread_mutex_destroy(&mHandle); - #endif - } -#endif - - /// Lock the mutex. - /// The method will block the calling thread until a lock on the mutex can - /// be obtained. The mutex remains locked until \c unlock() is called. - /// @see lock_guard - inline void lock() - { -#if defined(_FAST_MUTEX_ASM_) - bool gotLock; - do { - gotLock = try_lock(); - if(!gotLock) - { - #if defined(_TTHREAD_WIN32_) - Sleep(0); - #elif defined(_TTHREAD_POSIX_) - sched_yield(); - #endif - } - } while(!gotLock); -#else - #if defined(_TTHREAD_WIN32_) - EnterCriticalSection(&mHandle); - #elif defined(_TTHREAD_POSIX_) - pthread_mutex_lock(&mHandle); - #endif -#endif - } - - /// Try to lock the mutex. - /// The method will try to lock the mutex. If it fails, the function will - /// return immediately (non-blocking). - /// @return \c true if the lock was acquired, or \c false if the lock could - /// not be acquired. - inline bool try_lock() - { -#if defined(_FAST_MUTEX_ASM_) - int oldLock; - #if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) - asm volatile ( - "movl $1,%%eax\n\t" - "xchg %%eax,%0\n\t" - "movl %%eax,%1\n\t" - : "=m" (mLock), "=m" (oldLock) - : - : "%eax", "memory" - ); - #elif defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) - int *ptrLock = &mLock; - __asm { - mov eax,1 - mov ecx,ptrLock - xchg eax,[ecx] - mov oldLock,eax - } - #elif defined(__GNUC__) && (defined(__ppc__)) - int newLock = 1; - asm volatile ( - "\n1:\n\t" - "lwarx %0,0,%1\n\t" - "cmpwi 0,%0,0\n\t" - "bne- 2f\n\t" - "stwcx. %2,0,%1\n\t" - "bne- 1b\n\t" - "isync\n" - "2:\n\t" - : "=&r" (oldLock) - : "r" (&mLock), "r" (newLock) - : "cr0", "memory" - ); - #endif - return (oldLock == 0); -#else - #if defined(_TTHREAD_WIN32_) - return TryEnterCriticalSection(&mHandle) ? true : false; - #elif defined(_TTHREAD_POSIX_) - return (pthread_mutex_trylock(&mHandle) == 0) ? true : false; - #endif -#endif - } - - /// Unlock the mutex. - /// If any threads are waiting for the lock on this mutex, one of them will - /// be unblocked. - inline void unlock() - { -#if defined(_FAST_MUTEX_ASM_) - #if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) - asm volatile ( - "movl $0,%%eax\n\t" - "xchg %%eax,%0\n\t" - : "=m" (mLock) - : - : "%eax", "memory" - ); - #elif defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) - int *ptrLock = &mLock; - __asm { - mov eax,0 - mov ecx,ptrLock - xchg eax,[ecx] - } - #elif defined(__GNUC__) && (defined(__ppc__)) - asm volatile ( - "sync\n\t" // Replace with lwsync where possible? - : : : "memory" - ); - mLock = 0; - #endif -#else - #if defined(_TTHREAD_WIN32_) - LeaveCriticalSection(&mHandle); - #elif defined(_TTHREAD_POSIX_) - pthread_mutex_unlock(&mHandle); - #endif -#endif - } - - private: -#if defined(_FAST_MUTEX_ASM_) - int mLock; -#else - #if defined(_TTHREAD_WIN32_) - CRITICAL_SECTION mHandle; - #elif defined(_TTHREAD_POSIX_) - pthread_mutex_t mHandle; - #endif -#endif -}; - -} - -#endif // _FAST_MUTEX_H_ - diff --git a/depends/tthread/tinythread.h b/depends/tthread/tinythread.h index d575bae33..b5ff2f077 100644 --- a/depends/tthread/tinythread.h +++ b/depends/tthread/tinythread.h @@ -46,7 +46,6 @@ freely, subject to the following restrictions: /// @li tthread::recursive_mutex /// @li tthread::condition_variable /// @li tthread::lock_guard -/// @li tthread::fast_mutex /// /// @section misc_sec Miscellaneous /// The following special keywords are available: #thread_local. diff --git a/docs/Authors.rst b/docs/Authors.rst index 6821a84b0..6701b67a0 100644 --- a/docs/Authors.rst +++ b/docs/Authors.rst @@ -25,6 +25,7 @@ belal jimhester Ben Lubar BenLubar Ben Rosser TC01 billw2012 billw2012 +BrickViking brickviking brndd brndd burneddi Bumber Bumber64 Caldfir caldfir @@ -56,6 +57,7 @@ Hayati Ayguen hayguen Herwig Hochleitner bendlas Ian S kremlin- IndigoFenix +James Gilles kazimuth James Logsdon jlogsdon Japa JapaMala Jared Adams @@ -159,6 +161,7 @@ Timothy Collett danaris Timur Kelman TymurGubayev Tom Jobbins TheBloke Tom Prince +Tommy R tommy TotallyGatsby TotallyGatsby Travis Hoppe thoppe orthographic-pedant txtsd txtsd diff --git a/docs/Binpatches.rst b/docs/Binpatches.rst index a0fcc6630..b92190bc8 100644 --- a/docs/Binpatches.rst +++ b/docs/Binpatches.rst @@ -15,7 +15,8 @@ the `binpatch` command. at all possible - that way your work will work for many versions across multiple operating systems. -.. contents:: +.. contents:: Contents + :local: Getting a patch @@ -24,7 +25,7 @@ There are no binary patches available for Dwarf Fortress versions after 0.34.11. This system is kept for the chance that someone will find it useful, so some hints on how to write your own follow. This will require disassembly and -decent skill in `memory research `. +decent skill in `memory research `. * The patches are expected to be encoded in text format used by IDA. diff --git a/docs/Compile.rst b/docs/Compile.rst index f30775a03..68187e7fa 100644 --- a/docs/Compile.rst +++ b/docs/Compile.rst @@ -1,48 +1,82 @@ +.. _compile: + ################ Compiling DFHack ################ -You don't need to compile DFHack unless you're developing plugins or working on the core. +DFHack builds are available for all supported platforms; see `installing` for +installation instructions. If you are a DFHack end-user, modder, or plan on +writing scripts (not plugins), it is generally recommended (and easier) to use +these builds instead of compiling DFHack from source. -For users, modders, and authors of scripts it's better to download -and `install the latest release instead `. +However, if you are looking to develop plugins, work on the DFHack core, make +complex changes to DF-structures, or anything else that requires compiling +DFHack from source, this document will walk you through the build process. Note +that some steps may be unconventional compared to other projects, so be sure to +pay close attention if this is your first time compiling DFHack. -.. contents:: - :depth: 2 +.. contents:: Contents + :local: + :depth: 1 .. _compile-how-to-get-the-code: How to get the code =================== -DFHack doesn't have any kind of system of code snapshots in place, so you will have to -get code from the GitHub repository using Git. How to get Git is described under -the instructions for each platform. - -To get the latest release code (master branch):: +DFHack uses Git for source control; instructions for installing Git can be found +in the platform-specific sections below. The code is hosted on +`GitHub `_, and can be downloaded with:: git clone --recursive https://github.com/DFHack/dfhack cd dfhack -If your version of Git does not support the ``--recursive`` flag, you will need to omit it and run -``git submodule update --init`` after entering the dfhack directory. +If your version of Git does not support the ``--recursive`` flag, you will need +to omit it and run ``git submodule update --init`` after entering the dfhack +directory. -To get the latest development code (develop branch), clone as above and then:: +This will check out the code on the default branch of the GitHub repo, currently +``develop``, which may be unstable. If you want code for the latest stable +release, you can check out the ``master`` branch instead:: - git checkout develop + git checkout master git submodule update -Generally, you should only need to clone DFHack once. - -**Important note regarding submodule update after pulling or changing branches**: - -You must run ``git submodule update`` every time you change branches, such as -when switching between the master and develop branches or vice versa. You also -must run it after pulling any changes to submodules from the DFHack repo. If a -submodule only exists on the newer branch, or if a commit you just pulled -contains a new submodule, you need to run ``git submodule update --init``. -Failure to do this may result in a variety of errors, including ``fatal: -does not exist`` when using Git, errors when building DFHack, and ``not a known -DF version`` when starting DF. +In general, a single DFHack clone is suitable for development - most Git +operations such as switching branches can be done on an existing clone. If you +find yourself cloning DFHack frequently as part of your development process, or +getting stuck on anything else Git-related, feel free to reach out to us for +assistance. + +.. admonition:: A note on submodules + + DFHack uses submodules extensively to manage its subprojects (including the + ``scripts`` folder and DF-structures in ``library/xml``). Failing to keep + submodules in sync when switching between branches can result in build errors + or scripts that don't work. In general, you should always update submodules + whenever you switch between branches in the main DFHack repo with + ``git submodule update``. (If you are working on bleeding-edge DFHack and + have checked out the master branch of some submodules, running ``git pull`` + in those submodules is also an option.) + + Rarely, we add or remove submodules. If there are any changes to the existence + of submodules when you switch between branches, you should run + ``git submodule update --init`` instead (adding ``--init`` to the above + command). + + Some common errors that can arise when failing to update submodules include: + + * ``fatal: does not exist`` when performing Git operations + * Build errors, particularly referring to structures in the ``df::`` namespace + or the ``library/include/df`` folder + * ``Not a known DF version`` when starting DF + * ``Run 'git submodule update --init'`` when running CMake + + Submodules are a particularly confusing feature of Git. The + `Git Book `_ has a + thorough explanation of them (as well as of many other aspects of Git) and + is a recommended resource if you run into any issues. Other DFHack developers + are also able to help with any submodule-related (or Git-related) issues + you may encounter. **More notes**: @@ -51,18 +85,21 @@ DF version`` when starting DF. Contributing to DFHack ====================== -If you want to get involved with the development, create an account on -GitHub, make a clone there and then use that as your remote repository instead. +To contribute to DFHack on GitHub, you will need a GitHub account. Only some +DFHack developers can push directly to the DFHack repositories; we recommend +making a fork of whatever repos you are interested in contributing to, making +changes there, and submitting pull requests. `GitHub's pull request tutorial +`_ +is a good resource for getting started with pull requests (some things to note: +our work mainly happens on the ``develop`` branch, and you will need to use +your own fork, assuming that you don't have write access to the DFHack repos). -We'd love that; join us on IRC_ (#dfhack channel on freenode) for discussion, -and whenever you need help. +Most development-related discussion happens on IRC or in individual GitHub +issues and pull requests, but there are also other ways to reach out - see +`support` for details. -.. _IRC: https://webchat.freenode.net/?channels=dfhack - -(Note: for submodule issues, please see the above instructions first!) - -For lots more details on contributing to DFHack, including pull requests, code format, -and more, please see `contributing-code`. +For more details on contributing to DFHack, including pull requests, code +format, and more, please see `contributing-code`. Build settings @@ -78,7 +115,8 @@ Generator The ``Ninja`` CMake build generator is the prefered build method on Linux and macOS, instead of ``Unix Makefiles``, which is the default. You can select Ninja by passing ``-G Ninja`` to CMake. Incremental builds using Unix Makefiles can be -much slower than Ninja builds. +much slower than Ninja builds. Note that you will probably need to install +Ninja; see the platform-specific sections for details. :: @@ -126,6 +164,8 @@ in any case. Note that the scripts in the "build" folder on Windows will set the architecture automatically. +.. _compile-build-options: + Other settings -------------- There are a variety of other settings which you can find in CMakeCache.txt in @@ -133,6 +173,9 @@ your build folder or by running ``ccmake`` (or another CMake GUI). Most DFHack-specific settings begin with ``BUILD_`` and control which parts of DFHack are built. + +.. _compile-linux: + Linux ===== On Linux, DFHack acts as a library that shadows parts of the SDL API using LD_PRELOAD. @@ -144,13 +187,18 @@ DFHack is meant to be installed into an existing DF folder, so get one ready. We assume that any Linux platform will have ``git`` available (though it may need to be installed with your package manager.) -To build DFHack you need GCC version 4.8 or later. GCC 4.8 is easiest to work -with due to avoiding libstdc++ issues (see below), but any version from 4.8 -onwards (including 5.x) will work. +To build DFHack, you need GCC 4.8 or newer. GCC 4.8 has the benefit of avoiding +`libstdc++ compatibility issues `, but can be hard +to obtain on modern distributions, and working around these issues is done +automatically by the ``dfhack`` launcher script. As long as your system-provided +GCC is new enough, it should work. Note that extremely new GCC versions may not +have been used to build DFHack yet, so if you run into issues with these, please +let us know (e.g. by opening a GitHub issue). Before you can build anything, you'll also need ``cmake``. It is advisable to also get ``ccmake`` on distributions that split the cmake package into multiple -parts. +parts. As mentioned above, ``ninja`` is recommended (many distributions call +this package ``ninja-build``). You will need pthread; most systems should have this already. Note that older CMake versions may have trouble detecting pthread, so if you run into @@ -226,37 +274,42 @@ This will show a curses-based interface that lets you set all of the extra options. You can also use a cmake-friendly IDE like KDevelop 4 or the cmake-gui program. +.. _linux-incompatible-libstdcxx: + Incompatible libstdc++ ~~~~~~~~~~~~~~~~~~~~~~ -When compiling dfhack yourself, it builds against your system libstdc++. When -Dwarf Fortress runs, it uses a libstdc++ shipped with the binary, which comes -from GCC 4.8 and is incompatible with code compiled with newer GCC versions. If -you compile DFHack with a GCC version newer than 4.8, you will see an error -message such as:: +When compiling DFHack yourself, it builds against your system libstdc++. When +Dwarf Fortress runs, it uses a libstdc++ shipped in the ``libs`` folder, which +comes from GCC 4.8 and is incompatible with code compiled with newer GCC +versions. As of DFHack 0.42.05-alpha1, the ``dfhack`` launcher script attempts +to fix this by automatically removing the DF-provided libstdc++ on startup. +In rare cases, this may fail and cause errors such as:: ./libs/Dwarf_Fortress: /pathToDF/libs/libstdc++.so.6: version `GLIBCXX_3.4.18' not found (required by ./hack/libdfhack.so) -To fix this you can compile with GCC 4.8 or remove the libstdc++ shipped with +The easiest way to fix this is generally removing the libstdc++ shipped with DF, which causes DF to use your system libstdc++ instead:: cd /path/to/DF/ rm libs/libstdc++.so.6 -Note that distributing binaries compiled with newer GCC versions requires end- -users to delete libstdc++ themselves and have a libstdc++ on their system from -the same GCC version or newer. For this reason, distributing anything compiled -with GCC versions newer than 4.8 is discouraged. In the future we may start -bundling a later libstdc++ as part of the DFHack package, so as to enable -compilation-for-distribution with a GCC newer than 4.8. - -Mac OS X -======== -DFHack functions similarly on OS X and Linux, and the majority of the -information above regarding the build process (cmake and ninja) applies here +Note that distributing binaries compiled with newer GCC versions may result in +the opposite compatibily issue: users with *older* GCC versions may encounter +similar errors. This is why DFHack distributes both GCC 4.8 and GCC 7 builds. If +you are planning on distributing binaries to other users, we recommend using an +older GCC (but still at least 4.8) version if possible. + + +.. _compile-macos: + +macOS +===== +DFHack functions similarly on macOS and Linux, and the majority of the +information above regarding the build process (CMake and Ninja) applies here as well. -DFHack can officially be built on OS X with GCC 4.8 or 7. Anything newer than 7 +DFHack can officially be built on macOS only with GCC 4.8 or 7. Anything newer than 7 will require you to perform extra steps to get DFHack to run (see `osx-new-gcc-notes`), and your build will likely not be redistributable. @@ -309,7 +362,7 @@ Dependencies and system set-up cleaner, quicker, and smarter. For example, installing MacPort's GCC will install more than twice as many dependencies as Homebrew's will, and all in both 32-bit and 64-bit variants. Homebrew also doesn't require constant use - of sudo. + of ``sudo``. Using `Homebrew `_ (recommended):: @@ -378,9 +431,9 @@ Building export CC=gcc-7 export CXX=g++-7 - etc. + (adjust as needed for different GCC installations) -* Build dfhack:: +* Build DFHack:: mkdir build-osx cd build-osx @@ -390,6 +443,7 @@ Building should be a path to a copy of Dwarf Fortress, of the appropriate version for the DFHack you are building. + .. _compile-windows: Windows @@ -496,23 +550,25 @@ to your binary search PATH so the tool can be later run from anywhere. Perl / Strawberry Perl ^^^^^^^^^^^^^^^^^^^^^^ -For the code generation parts you'll need Perl 5 with XML::LibXML and XML::LibXSLT. -`Strawberry Perl `_ is recommended as it includes -all of the required packages in a single, easy install. +For the code generation stage of the build process, you'll need Perl 5 with +XML::LibXML and XML::LibXSLT. `Strawberry Perl `_ is +recommended as it includes all of the required packages in a single, easy +install. After install, ensure Perl is in your user's PATH. This can be edited from ``Control Panel -> System -> Advanced System Settings -> Environment Variables``. -The following three directories must be in PATH, in this order: +The following directories must be in your PATH, in this order: * ``\c\bin`` * ``\perl\site\bin`` * ``\perl\bin`` +* ``\perl\vendor\lib\auto\XML\LibXML`` (may only be required on some systems) Be sure to close and re-open any existing ``cmd.exe`` windows after updating your PATH. -If you already have a different version of Perl (for example the one from Cygwin), +If you already have a different version of Perl installed (for example, from Cygwin), you can run into some trouble. Either remove the other Perl install from PATH, or install XML::LibXML and XML::LibXSLT for it using CPAN. @@ -607,129 +663,8 @@ Then build the ``INSTALL`` target listed under ``CMakePredefinedTargets``. Building the documentation ========================== -DFHack documentation, like the file you are reading now, is created as .rst files, -which are in `reStructuredText (reST) `_ format. -This is a documenation format that has come from the Python community. It is very -similar in concept - and in syntax - to Markdown, as found on GitHub and many other -places. However it is more advanced than Markdown, with more features available when -compiled to HTML, such as automatic tables of contents, cross-linking, special -external links (forum, wiki, etc) and more. The documentation is compiled by a -Python tool, `Sphinx `_. - -The DFHack build process will compile the documentation but this has been disabled -by default. You only need to build the docs if you're changing them, or perhaps -if you want a local HTML copy; otherwise, read them easily online at -`ReadTheDoc's DFHack hosted documentation `_. - -(Note that even if you do want a local copy, it is certainly not necesesary to -compile the documentation in order to read it. Like Markdown, reST documents are -designed to be just as readable in a plain-text editor as they are in HTML format. -The main thing you lose in plain text format is hyperlinking.) - - -Enabling documentation building -------------------------------- -First, make sure you have followed all the necessary steps for your platform as -outlined in the rest of this document. - -To compile documentation with DFHack, add the following flag to your ``cmake`` command:: - - -DBUILD_DOCS:bool=ON - -For example:: - - cmake .. -DCMAKE_BUILD_TYPE:string=Release -DBUILD_DOCS:bool=ON -DCMAKE_INSTALL_PREFIX= - -Alternatively you can use the CMake GUI which allows options to be changed easily. - -On Windows you should either use ``generate-msvc-gui.bat`` and set the option -through the GUI, or else if you want to use an alternate file, such as -``generate-msvc-all.bat``, you will need to edit it to add the flag. -Or you could just run ``cmake`` on the command line like in other platforms. - -Required dependencies ---------------------- -In order to build the documentation, you must have Python with Sphinx -version 1.3.1 or later. Both Python 2.x and 3.x are supported. - -When installing Sphinx from OS package managers, be aware that there is -another program called Sphinx, completely unrelated to documentation management. -Be sure you are installing the right Sphinx; it may be called ``python-sphinx``, -for example. To avoid doubt, ``pip`` can be used instead as detailed below. - - -Linux ------ -Most Linux distributions will include Python as standard. - -Check your package manager to see if Sphinx 1.3.1 or later is available, -but at the time of writing Ubuntu for example only has 1.2.x. - -You can instead install Sphinx with the pip package manager. This may need -to be installed from your OS package manager; this is the case on Ubuntu. -On Ubuntu/Debian, use the following to first install pip:: - - sudo apt-get install python-pip - -Once pip is available, you can then install the Python Sphinx module with:: - - pip install sphinx - -If you run this as a normal user it will install a local copy for your user only. -Run it with sudo if you want a system-wide install. Either is fine for DFHack, -however if installing locally do check that ``sphinx-build`` is in your path. -It may be installed in a directory such as ``~/.local/bin/``, so after pip -install, find ``sphinx-build`` and ensure its directory is in your local ``$PATH``. - - -Mac OS X --------- -OS X has Python 2.7 installed by default, but it does not have the pip package manager. - -You can install Homebrew's Python 3, which includes pip, and then install the -latest Sphinx using pip:: - - brew install python3 - pip3 install sphinx - -Alternatively, you can simply install Sphinx 1.3.x directly from Homebrew:: - - brew install sphinx-doc - -This will install Sphinx for OS X's system Python 2.7, without needing pip. - -Either method works; if you plan to use Python for other purposes, it might best -to install Homebrew's Python 3 so that you have the latest Python as well as pip. -If not, just installing sphinx-doc for OS X's system Python 2.7 is fine. - - -Windows -------- -Use the Chocolatey package manager to install Python and pip, -then use pip to install Sphinx. - -Run the following commands from an elevated (Admin) ``cmd.exe``, after installing -Chocolatey as outlined in the `Windows section `:: - - choco install python pip -y - -Then close that Admin ``cmd.exe``, re-open another Admin ``cmd.exe``, and run:: - - pip install sphinx - -.. _build-changelog: - -Building the changelogs ------------------------ -If you have Python installed, but do not want to build all of the documentation, -you can build the changelogs with the ``docs/gen_changelog.py`` script. - -All changes should be listed in ``changelog.txt``. A description of this file's -format follows: - -.. include:: /docs/changelog.txt - :start-after: ===help - :end-before: ===end +The steps above will not build DFHack's documentation by default. If you are +editing documentation, see `documentation` for details on how to build it. Misc. Notes =========== @@ -767,7 +702,7 @@ files alphabetically, so all the files you need should be next to each other. It is recommended that you create a build folder and run CMake to verify that you have downloaded everything at this point, assuming your download machine has CMake installed. This involves running a "generate" batch script on Windows, or -a command starting with ``cmake .. -G Ninja`` on Linux and OS X, following the +a command starting with ``cmake .. -G Ninja`` on Linux and macOS, following the instructions in the sections above. CMake should automatically locate files that you placed in ``CMake/downloads``, and use them instead of attempting to download them. diff --git a/docs/Contributing.rst b/docs/Contributing.rst new file mode 100644 index 000000000..ab92d0571 --- /dev/null +++ b/docs/Contributing.rst @@ -0,0 +1,59 @@ +.. _contributing: + +########################### +How to contribute to DFHack +########################### + +.. contents:: Contents + :local: + +.. _contributing-code: + +Contributing Code +================= +Several things should be kept in mind when contributing code to DFHack. + +Code Format +----------- +* Four space indents for C++. Never use tabs for indentation in any language. +* LF (Unix style) line terminators +* Avoid trailing whitespace +* UTF-8 encoding +* For C++: + + * Opening and closing braces on their own lines or opening brace at the end of the previous line + * Braces placed at original indent level if on their own lines + * #includes should be sorted. C++ libraries first, then dfhack modules, then df structures, + then local includes. Within each category they should be sorted alphabetically. + +How to get new code into DFHack +------------------------------- +* Submit pull requests to the ``develop`` branch, not the ``master`` branch. + (The ``master`` branch always points at the most recent release) +* Use a new branch for each feature or bugfix so that your changes can be merged independently + (i.e. not the master or develop branch of your fork). +* If possible, compile on multiple platforms when changing anything that compiles +* It must pass CI - run ``python travis/all.py`` to check this. +* Update documentation when applicable - see `docs-standards` for details. +* Update ``changelog.txt`` and ``docs/Authors.rst`` when applicable. See + `build-changelog` for more information on the changelog format. +* Create a GitHub pull request once finished +* Submit ideas and bug reports as :issue:`issues on GitHub <>`. + Posts in the forum thread can easily get missed or forgotten. +* Work on :issue:`reported problems ` + will take priority over ideas or suggestions. + + +Other ways to help +================== +DFHack is a software project, but there's a lot more to it than programming. +If you're not comfortable programming, you can help by: + +* reporting bugs and incomplete documentation +* improving the documentation +* finding third-party scripts to add +* writing tutorials for newbies + +All those things are crucial, and often under-represented. So if that's +your thing, go get started! + diff --git a/docs/Core.rst b/docs/Core.rst index 4e65f4dda..6ef1d0575 100644 --- a/docs/Core.rst +++ b/docs/Core.rst @@ -4,8 +4,9 @@ DFHack Core ########### -.. contents:: - :depth: 2 +.. contents:: Contents + :local: + :depth: 2 Command Implementation @@ -165,10 +166,13 @@ right place to do it. Most such plugins or scripts support the built-in ``enable`` and ``disable`` commands. Calling them at any time without arguments prints a list of enabled and disabled plugins, and shows whether that can be changed -through the same commands. +through the same commands. Passing plugin names to these commands will enable +or disable the specified plugins. For example, to enable the `manipulator` +plugin:: + + enable manipulator -To enable or disable plugins that support this, use their names as -arguments for the command:: +It is also possible to enable or disable multiple plugins at once:: enable manipulator search @@ -271,6 +275,9 @@ something. Usage:: Allows dealing with plugins individually by name, or all at once. +Note that plugins do not maintain their enabled state if they are reloaded, so +you may need to use `enable` to re-enable a plugin after reloading it. + .. _ls: @@ -408,7 +415,7 @@ All matching init files will be executed in alphebetical order. A world being loaded can mean a fortress, an adventurer, or legends mode. These files are best used for non-persistent commands, such as setting -a `fix ` script to run on `repeat`. +a `fix ` script to run on `repeat`. .. _onUnload.init: diff --git a/docs/Dev-intro.rst b/docs/Dev-intro.rst new file mode 100644 index 000000000..9a08533c0 --- /dev/null +++ b/docs/Dev-intro.rst @@ -0,0 +1,87 @@ +=========================== +DFHack development overview +=========================== + +DFHack has various components; this page provides an overview of some. If you +are looking to develop a tool for DFHack, developing a script or plugin is +likely the most straightforward choice. + +Other pages that may be relevant include: + +- `contributing` +- `documentation` +- `license` + + +.. contents:: Contents + :local: + + +Plugins +------- + +DFHack plugins are written in C++ and located in the ``plugins`` folder. +Currently, documentation on how to write plugins is somewhat sparse. There are +templates that you can get use to get started in the ``plugins/skeleton`` +folder, and the source code of existing plugins can also be helpful. + +If you want to compile a plugin that you have just added, you will need to add a +call to ``DFHACK_PLUGIN`` in ``plugins/CMakeLists.txt``. + +Plugins have the ability to make one or more commands available to users of the +DFHack console. Examples include `3dveins` (which implements the ``3dveins`` +command) and `reveal` (which implements ``reveal``, ``unreveal``, and several +other commands). + +Plugins can also register handlers to run on every tick, and can interface with +the built-in `enable` and `disable` commands. For the full plugin API, see the +skeleton plugins or ``PluginManager.cpp``. + +Installed plugins live in the ``hack/plugins`` folder of a DFHack installation, +and the `load` family of commands can be used to load a recompiled plugin +without restarting DF. + +See `plugins-index` for a list of all plugins included in DFHack. + +Scripts +------- + +DFHack scripts can currently be written in Lua or Ruby. The `Lua API ` +is more complete and currently better-documented, however. Referring to existing +scripts as well as the API documentation can be helpful when developing new +scripts. + +`Scripts included in DFHack ` live in a separate `scripts repository `_. +This can be found in the ``scripts`` submodule if you have +`cloned DFHack `, or the ``hack/scripts`` folder +of an installed copy of DFHack. + +Core +---- + +The `DFHack core ` has a variety of low-level functions. It is +responsible for hooking into DF (via SDL), providing a console, and providing an +interface for plugins and scripts to interact with DF. + +Modules +------- + +A lot of shared code to interact with DF in more complicated ways is contained +in **modules**. For example, the Units module contains functions for checking +various traits of units, changing nicknames properly, and more. Generally, code +that is useful to multiple plugins and scripts should go in the appropriate +module, if there is one. + +Several modules are also `exposed to Lua `, although +some functions (and some entire modules) are currently only available in C++. + +Remote access interface +----------------------- +DFHack supports remote access by exchanging Google protobuf messages via a TCP +socket. Both the core and plugins can define remotely accessible methods. The +``dfhack-run`` command uses this interface to invoke ordinary console commands. + +Currently the supported set of requests is limited, because the developers don't +know what exactly is most useful. `remotefortressreader` provides a fairly +comprehensive interface for visualisers such as :forums:`Armok Vision <146473>`. + diff --git a/docs/Documentation.rst b/docs/Documentation.rst new file mode 100644 index 000000000..462932b0f --- /dev/null +++ b/docs/Documentation.rst @@ -0,0 +1,285 @@ +.. _documentation: + +########################### +DFHack Documentation System +########################### + + +DFHack documentation, like the file you are reading now, is created as ``.rst`` files, +which are in `reStructuredText (reST) `_ format. +This is a documentation format common in the Python community. It is very +similar in concept - and in syntax - to Markdown, as found on GitHub and many other +places. However it is more advanced than Markdown, with more features available when +compiled to HTML, such as automatic tables of contents, cross-linking, special +external links (forum, wiki, etc) and more. The documentation is compiled by a +Python tool, `Sphinx `_. + +The DFHack build process will compile the documentation, but this is disabled +by default due to the additional Python and Sphinx requirements. You typically +only need to build the docs if you're changing them, or perhaps +if you want a local HTML copy; otherwise, you can read an +`online version hosted by ReadTheDocs `_. + +(Note that even if you do want a local copy, it is certainly not necessary to +compile the documentation in order to read it. Like Markdown, reST documents are +designed to be just as readable in a plain-text editor as they are in HTML format. +The main thing you lose in plain text format is hyperlinking.) + +.. contents:: Contents + :local: + +.. _docs-standards: + +Documentation standards +======================= + +Whether you're adding new code or just fixing old documentation (and there's plenty), +there are a few important standards for completeness and consistent style. Treat +this section as a guide rather than iron law, match the surrounding text, and you'll +be fine. + +Each command should have a short (~54 character) help string, which is shown +by the `ls` command. For scripts, this is a comment on the first line +(the comment marker and whitespace is stripped). For plugins it's the second +argument to ``PluginCommand``. Please make this brief but descriptive! + +Everything should be documented! If it's not clear *where* a particular +thing should be documented, ask on IRC or in the DFHack thread on Bay12 - +as well as getting help, you'll be providing valuable feedback that +makes it easier for future readers! + +Scripts can use a custom autodoc function, based on the Sphinx ``include`` +directive - anything between the tokens is copied into the appropriate scripts +documentation page. For Ruby, we follow the built-in docstring convention +(``=begin`` and ``=end``). For Lua, the tokens are ``[====[`` and ``]====]`` +- ordinary multi-line strings. It is highly encouraged to reuse this string +as the in-console documentation by (eg.) printing it when a ``-help`` argument +is given. + +The docs **must** have a heading which exactly matches the command, underlined +with ``=====`` to the same length. For example, a lua file would have:: + + local helpstr = [====[ + + add-thought + =========== + Adds a thought or emotion to the selected unit. Can be used by other scripts, + or the gui invoked by running ``add-thought gui`` with a unit selected. + + ]====] + + +Where the heading for a section is also the name of a command, the spelling +and case should exactly match the command to enter in the DFHack command line. + +Try to keep lines within 80-100 characters, so it's readable in plain text +in the terminal - Sphinx (our documentation system) will make sure +paragraphs flow. + +If there aren't many options or examples to show, they can go in a paragraph of +text. Use double-backticks to put commands in monospaced font, like this:: + + You can use ``cleanowned scattered x`` to dump tattered or abandoned items. + +If the command takes more than three arguments, format the list as a table +called Usage. The table *only* lists arguments, not full commands. +Input values are specified in angle brackets. Example:: + + Usage: + + :arg1: A simple argument. + :arg2 : Does something based on the input value. + :Very long argument: + Is very specific. + +To demonstrate usage - useful mainly when the syntax is complicated, list the +full command with arguments in monospaced font, then indent the next line and +describe the effect:: + + ``resume all`` + Resumes all suspended constructions. + +If it would be helpful to mention another DFHack command, don't just type the +name - add a hyperlink! Specify the link target in backticks, and it will be +replaced with the corresponding title and linked: eg ```autolabor``` +=> `autolabor`. Link targets should be equivalent to the command +described (without file extension), and placed above the heading of that +section like this:: + + .. _autolabor: + + autolabor + ========= + +Add link targets if you need them, but otherwise plain headings are preferred. +Scripts have link targets created automatically. + + +Building the documentation +========================== + +Required dependencies +--------------------- +In order to build the documentation, you must have Python with Sphinx +version |sphinx_min_version| or later. Both Python 2.x and 3.x are supported. + +When installing Sphinx from OS package managers, be aware that there is +another program called Sphinx, completely unrelated to documentation management. +Be sure you are installing the right Sphinx; it may be called ``python-sphinx``, +for example. To avoid doubt, ``pip`` can be used instead as detailed below. + +Once you have installed Sphinx, ``sphinx-build --version`` should report the +version of Sphinx that you have installed. If this works, CMake should also be +able to find Sphinx. + +For more detailed platform-specific instructions, see the sections below: + +.. contents:: + :local: + :backlinks: none + + +Linux +~~~~~ +Most Linux distributions will include Python by default. If not, start by +installing Python (preferably Python 3). On Debian-based distros:: + + sudo apt install python3 + +Check your package manager to see if Sphinx |sphinx_min_version| or later is +available. On Debian-based distros, this package is named ``python3-sphinx``. +If this package is new enough, you can install it directly. If not, or if you +want to use a newer Sphinx version (which may result in faster builds), you +can install Sphinx through the ``pip`` package manager instead. On Debian-based +distros, you can install pip with:: + + sudo apt install python3-pip + +Once pip is available, you can then install Sphinx with:: + + pip3 install sphinx + +If you run this as an unpriviliged user, it may install a local copy of Sphinx +for your user only. The ``sphinx-build`` executable will typically end up in +``~/.local/bin/`` in this case. Alternatively, you can install Sphinx +system-wide by running pip with ``sudo``. In any case, you will need the folder +containing ``sphinx-build`` to be in your ``$PATH``. + +macOS +~~~~~ +macOS has Python 2.7 installed by default, but it does not have the pip package manager. + +You can install Homebrew's Python 3, which includes pip, and then install the +latest Sphinx using pip:: + + brew install python3 + pip3 install sphinx + +Alternatively, you can simply install Sphinx directly from Homebrew:: + + brew install sphinx-doc + +This will install Sphinx for macOS's system Python 2.7, without needing pip. + +Either method works; if you plan to use Python for other purposes, it might best +to install Homebrew's Python 3 so that you have the latest Python as well as pip. +If not, just installing sphinx-doc for macOS's system Python 2.7 is fine. + + +Windows +~~~~~~~ +Python for Windows can be downloaded `from python.org `_. +The latest version of Python 3 is recommended, as it includes pip already. + +You can also install Python and pip through the Chocolatey package manager. +After installing Chocolatey as outlined in the `Windows compilation instructions `, +run the following command from an elevated (admin) command prompt (e.g. ``cmd.exe``):: + + choco install python pip -y + +Once you have pip available, you can install Sphinx with the following command:: + + pip install sphinx + +Note that this may require opening a new (admin) command prompt if you just +installed pip from the same command prompt. + + +Using CMake +----------- + +Enabling the ``BUILD_DOCS`` CMake option will cause the documentation to be built +whenever it changes as part of the normal DFHack build process. There are several +ways to do this: + +* When initially running CMake, add ``-DBUILD_DOCS:bool=ON`` to your ``cmake`` + command. For example:: + + cmake .. -DCMAKE_BUILD_TYPE:string=Release -DBUILD_DOCS:bool=ON -DCMAKE_INSTALL_PREFIX= + +* If you have already run CMake, you can simply run it again from your build + folder to update your configuration:: + + cmake .. -DBUILD_DOCS:bool=ON + +* You can edit the ``BUILD_DOCS`` setting in CMakeCache.txt directly + +* You can use the CMake GUI to change ``BUILD_DOCS`` + +* On Windows, if you prefer to use the batch scripts, you can run + ``generate-msvc-gui.bat`` and set ``BUILD_DOCS`` through the GUI. If you are + running another file, such as ``generate-msvc-all.bat``, you will need to edit + it to add the flag. You can also run ``cmake`` on the command line, similar to + other platforms. + +Running Sphinx manually +----------------------- + +You can also build the documentation without going through CMake, which may be +faster. There is a ``docs/build.sh`` script available for Linux and macOS that +will run essentially the same command that CMake runs - see the script for +options. + +To build the documentation with default options, run the following command from +the root DFHack folder:: + + sphinx-build . docs/html + +Sphinx has many options to enable clean builds, parallel builds, logging, and +more - run ``sphinx-build --help`` for details. + + +.. _build-changelog: + +Building the changelogs +======================= +If you have Python installed, but do not want to build all of the documentation, +you can build the changelogs with the ``docs/gen_changelog.py`` script. This +script provides additional options, including one to build individual changelogs +for all DFHack versions - run ``python docs/gen_changelog.py --help`` for details. + +Changelog entries are obtained from ``changelog.txt`` files in multiple repos. +This allows changes to be listed in the same repo where they were made. These +changelogs are combined as part of the changelog build process: + +* ``docs/changelog.txt`` for changes in the main ``dfhack`` repo +* ``scripts/changelog.txt`` for changes made to scripts in the ``scripts`` repo +* ``library/xml/changelog.txt`` for changes made in the ``df-structures`` repo + +Building the changelogs generates two files: ``docs/_auto/news.rst`` and +``docs/_auto/news-dev.rst``. These correspond to `changelog` and `dev-changelog` +and contain changes organized by stable and development DFHack releases, +respectively. For example, an entry listed under "0.44.05-alpha1" in +changelog.txt will be listed under that version in the development changelog as +well, but under "0.44.05-r1" in the stable changelog (assuming that is the +closest stable release after 0.44.05-alpha1). An entry listed under a stable +release like "0.44.05-r1" in changelog.txt will be listed under that release in +both the stable changelog and the development changelog. + + +Changelog syntax +---------------- + +.. include:: /docs/changelog.txt + :start-after: ===help + :end-before: ===end diff --git a/docs/History.rst b/docs/History.rst index 8386c4cab..d382b351e 100644 --- a/docs/History.rst +++ b/docs/History.rst @@ -1,16 +1,19 @@ :orphan: -.. _HISTORY: +.. _History: -######################## -HISTORY - old changelogs -######################## +##################### +Historical changelogs +##################### -This file is where old changelogs live, so the current `changelog` -in ``NEWS.rst`` doesn't get too long. +This file is where old changelogs live, so the `current changelog ` +doesn't get too long. Some of these changelogs are also formatted differently +from current changelogs and would be difficult for the current `changelog +generation system ` to handle. -.. contents:: - :depth: 2 +.. contents:: Contents + :local: + :depth: 1 DFHack 0.43.05-r3 ================= diff --git a/docs/Introduction.rst b/docs/Introduction.rst index 3a36aa538..4490086f3 100644 --- a/docs/Introduction.rst +++ b/docs/Introduction.rst @@ -7,30 +7,32 @@ Introduction and Overview DFHack is a Dwarf Fortress memory access library, distributed with a wide variety of useful scripts and plugins. -The project is currently hosted at https://www.github.com/DFHack/dfhack, -and can be downloaded from `the releases page -`_. +The project is currently hosted `on GitHub `_, +and can be downloaded from `the releases page `_ +- see `installing` for installation instructions. This is also where the +`DFHack bug tracker `_ is hosted. -All new releases are announced in :forums:`the bay12 forums thread <139553>`, +All new releases are announced in `the Bay12 forums thread `_, which is also a good place for discussion and questions. -For users, it provides a significant suite of bugfixes and interface +For users, DFHack provides a significant suite of bugfixes and interface enhancements by default, and more can be enabled. There are also many tools (such as `workflow` or `autodump`) which can make life easier. You can even add third-party scripts and plugins to do almost anything! For modders, DFHack makes many things possible. Custom reactions, new -interactions, magic creature abilities, and more can be set through `modtools` +interactions, magic creature abilities, and more can be set through `scripts-modtools` and custom raws. Non-standard DFHack scripts and inits can be stored in the raw directory, making raws or saves fully self-contained for distribution - or for coexistence in a single DF install, even with incompatible components. For developers, DFHack unites the various ways tools access DF memory and allows easier development of new tools. As an open-source project under -`various copyleft licences `, contributions are welcome. +`various open-source licences `, contributions are welcome. -.. contents:: +.. contents:: Contents + :local: .. _installing: @@ -87,6 +89,7 @@ the console. .. _troubleshooting: +.. _support: Troubleshooting =============== @@ -101,7 +104,7 @@ If the search function in this documentation isn't enough and :wiki:`the DF Wiki <>` hasn't helped, try asking in: - the `#dfhack IRC channel on freenode `_ -- the :forums:`Bay12 DFHack thread <139553>` +- the `Bay12 DFHack thread `_ - the `/r/dwarffortress `_ questions thread - the thread for the mod or Starter Pack you're using (if any) diff --git a/docs/Lua API.rst b/docs/Lua API.rst index d7e94b92a..4c7163623 100644 --- a/docs/Lua API.rst +++ b/docs/Lua API.rst @@ -24,8 +24,9 @@ implemented by Lua files located in :file:`hack/lua/*` (:file:`library/lua/*` in the git repo). -.. contents:: - :depth: 3 +.. contents:: Contents + :local: + :depth: 2 ========================= @@ -801,6 +802,8 @@ Random number generation Dimension may be 1, 2 or 3 (default). +.. _lua-cpp-func-wrappers: + C++ function wrappers ===================== diff --git a/docs/Memory-research.rst b/docs/Memory-research.rst new file mode 100644 index 000000000..41aa437c8 --- /dev/null +++ b/docs/Memory-research.rst @@ -0,0 +1,149 @@ +.. _memory-research: + +############### +Memory research +############### + +There are a variety of tools that can be used to analyze DF memory - some are +listed here. Note that some of these may be old and unmaintained. If you aren't +sure what tool would be best for your purposes, feel free to ask for advice (on +IRC, Bay12, etc.). + +.. contents:: Contents + :local: + + +Cross-platform tools +==================== + +Ghidra +------ + +Ghidra is a cross-platform reverse-engineering framework (written in Java) +available at https://ghidra-sre.org. It supports analyzing both 32-bit and +64-bit executables for all supported DF platforms. There are some custom DFHack +Ghidra scripts available in the `df_misc`_ repo (look for ``.java`` files). + + +IDA Freeware 7.0 +---------------- + +Available from `Hex-Rays `_. +Supports analyzing both 32-bit and 64-bit executables for all supported DF platforms. +Some ``.idc`` scripts for IDA are available in the `df_misc`_ repo. + +.. _df_misc: https://github.com/DFHack/df_misc + + +Hopper +------ + +Runs on macOS and some Linux distributions; available from https://www.hopperapp.com/. +`TWBT `_ uses this to produce some patches. + + +DFHack tools +------------ + +Plugins +~~~~~~~ + +There are a few development plugins useful for low-level memory research. They +are not built by default, but can be built by setting the ``BUILD_DEVEL`` +`CMake option `. These include: + +- ``check-structures-sanity``, which performs sanity checks on the given DF + object. Note that this will crash in several cases, some intentional, so using + this with `GDB ` is recommended. +- ``memview``, which produces a hex dump of a given memory range. It also + highlights valid pointers, and can be configured to work with `sizecheck` + to auto-detect object sizes. +- ``vectors``, which can identify instances of ``std::vector`` in a given memory range. + +Scripts +~~~~~~~ + +Several `development scripts ` can be useful for memory research. +These include (but are not limited to): + +- `devel/dump-offsets` +- `devel/find-offsets` +- `devel/lsmem` +- `devel/sc` (requires `sizecheck`) +- `devel/visualize-structure` +- Generally, any script starting with ``devel/find`` + +.. _sizecheck: + +Sizecheck +~~~~~~~~~ + +Sizecheck is a custom tool that hooks into the memory allocator and inserts a +header indicating the size of every object. The corresponding logic to check for +this header when freeing memory usually works, but is inherently not foolproof. +You should not count on DF being stable when using this. + +DFHack's implementation of sizecheck is currently only tested on Linux, although +it probably also works on macOS. It can be built with the ``BUILD_SIZECHECK`` +`CMake option `, which produces a ``libsizecheck`` +library installed in the ``hack`` folder. You will need to preload this library +manually, by setting ``PRELOAD_LIB`` on Linux (or ``LD_PRELOAD`` if editing +the ``dfhack`` launcher script directly), or by editing the ``dfhack`` +launcher script and adding the library to ``DYLD_INSERT_LIBRARIES`` on macOS. + +There is also an older sizecheck implementation by Mifki available on +`GitHub `__ (``b.cpp`` is the main +sizecheck library, and ``win_patch.cpp`` is used for Windows support). To use +this with other DFHack tools, you will likely need to edit the header's +magic number to match what is used in `devel/sc` (search for a hexadecimal +constant starting with ``0x``). + +Legacy tools +~~~~~~~~~~~~ + +Some very old DFHack tools are available in the `legacy branch on GitHub `_. +No attempt is made to support these. + + +Linux-specific tools +==================== + +.. _linux-gdb: + +GDB +--- + +`GDB `_ is technically cross-platform, but +tends to work best on Linux, and DFHack currently only offers support for using +GDB on 64-bit Linux. To start with GDB, pass ``-g`` to the DFHack launcher +script:: + + ./dfhack -g + +Some basic GDB commands: + +- ``run``: starts DF from the GDB prompt. Any arguments will be passed as + command-line arguments to DF (e.g. `load-save` may be useful). +- ``bt`` will produce a backtrace if DF crashes. + +See the `official GDB documentation `_ +for more details. + + +df-structures GUI +----------------- + +This is a tool written by Angavrilov and available on `GitHub `__. +It only supports 32-bit DF. Some assistance may be available on IRC. + + +EDB (Evan's debugger) +--------------------- + +Available on `GitHub `__. + + +Windows-specific tools +====================== + +Some people have used `Cheat Engine `__ for research in the past. diff --git a/docs/NEWS-dev.rst b/docs/NEWS-dev.rst index 7a9da7cb7..f0c47fe96 100644 --- a/docs/NEWS-dev.rst +++ b/docs/NEWS-dev.rst @@ -13,7 +13,8 @@ which they first appeared. See `build-changelog` for more information. See `changelog` for a list of changes grouped by stable releases. -.. contents:: - :depth: 2 +.. contents:: Contents + :local: + :depth: 1 .. include:: /docs/_auto/news-dev.rst diff --git a/docs/NEWS.rst b/docs/NEWS.rst index 22f9881b9..1283b079a 100644 --- a/docs/NEWS.rst +++ b/docs/NEWS.rst @@ -13,14 +13,15 @@ appeared. See `build-changelog` for more information. See `dev-changelog` for a list of changes grouped by development releases. -.. contents:: - :depth: 2 +.. contents:: Contents + :local: + :depth: 1 .. include:: /docs/_auto/news.rst Older Changelogs ================ -Are kept in a seperate file: `HISTORY` +Are kept in a seperate file: `History` -.. that's ``docs/history.rst``, if you're reading the raw text. +.. that's ``docs/History.rst``, if you're reading the raw text. diff --git a/docs/Plugins.rst b/docs/Plugins.rst index 421e5e81b..0185dbee5 100644 --- a/docs/Plugins.rst +++ b/docs/Plugins.rst @@ -1,3 +1,5 @@ +.. _plugins-index: + ############## DFHack Plugins ############## @@ -9,8 +11,9 @@ game subsystems or the entire renderer. Most commands offered by plugins are listed here, hopefully organised in a way you will find useful. -.. contents:: - :depth: 3 +.. contents:: Contents + :local: + :depth: 2 =============================== Data inspection and visualizers @@ -2356,11 +2359,15 @@ dfhack command line and can't be used from a hotkey. Settings will be remembered as long as dfhack runs. Intended for use in combination with the command ``liquids-here`` (which can be bound to a hotkey). See also :issue:`80`. -.. note:: +.. warning:: Spawning and deleting liquids can mess up pathing data and temperatures (creating heat traps). You've been warned. +.. note:: + + `gui/liquids` is an in-game UI for this script. + Settings will be remembered until you quit DF. You can call `liquids-here` to execute the last configured action, which is useful in combination with keybindings. diff --git a/docs/Removed.rst b/docs/Removed.rst new file mode 100644 index 000000000..f98915bdd --- /dev/null +++ b/docs/Removed.rst @@ -0,0 +1,17 @@ +############# +Removed tools +############# + +This page lists tools (plugins or scripts) that were previously included in +DFHack but have been removed. It exists primarily so that internal links still +work (e.g. links from the `changelog`). + +.. contents:: Contents + :local: + :depth: 1 + +.. _warn-stuck-trees: + +warn-stuck-trees +================ +The corresponding DF bug, :bug:`9252` was fixed in DF 0.44.01. diff --git a/docs/Scripts-removed.rst b/docs/Scripts-removed.rst deleted file mode 100644 index 4490110ba..000000000 --- a/docs/Scripts-removed.rst +++ /dev/null @@ -1,14 +0,0 @@ -############### -Removed scripts -############### - -The following scripts were removed for various reasons. - -.. contents:: - :depth: 2 - -.. _warn-stuck-trees: - -warn-stuck-trees -================ -The corresponding DF bug, :bug:`9252` was fixed in DF 0.44.01. diff --git a/docs/Scripts.rst b/docs/Scripts.rst index 0e6ef415c..419385582 100644 --- a/docs/Scripts.rst +++ b/docs/Scripts.rst @@ -1,3 +1,5 @@ +.. _scripts-index: + ############## DFHack Scripts ############## diff --git a/docs/Structures-intro.rst b/docs/Structures-intro.rst new file mode 100644 index 000000000..c45d03eaf --- /dev/null +++ b/docs/Structures-intro.rst @@ -0,0 +1,34 @@ +=================================== +DF data definitions (DF-structures) +=================================== + +DFHack's information about DF's data structures is stored in XML files in the +`df-structures repository `_. If you +have `obtained a local copy of the DFHack source `, +DF-structures is included as a submodule in ``library/xml``. + +Data structure layouts are described in files named with the ``df.*.xml`` +pattern. This information is transformed by a Perl script (``codegen.pl``) into +C++ headers, as well as metadata for the Lua wrapper. This ultimately allows +DFHack code to access DF data directly, with the same speed and capabilities as +DF itself, which is an advantage over the older out-of-process approach (used +by debuggers and utilities like Dwarf Therapist). The main disadvantage of this +approach is that any compiled code relying on these layouts will break when DF's +layout changes, and will need to be recompiled for every new DF version. + +Addresses of DF global objects and vtables are stored in a separate file, +:file:`symbols.xml`. Since these are only absolute addresses, they do not need +to be compiled in to DFHack code, and are instead loaded at runtime. This makes +fixes and additions to global addresses possible without recompiling DFHack. +In an installed copy of DFHack, this file can be found at the root of the +``hack`` folder. + +The following pages contain more detailed information about various aspects +of DF-structures: + +.. toctree:: + :titlesonly: + + /library/xml/SYNTAX + /library/xml/how-to-update + diff --git a/docs/build.sh b/docs/build.sh index 6745db4d5..48e1bfd4a 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -17,4 +17,4 @@ if [ -z "$JOBS" ]; then JOBS=2 fi -"$sphinx" -a -E -q -b html . ./docs/html -w ./docs/_sphinx-warnings.txt -j "$JOBS" "$@" +"$sphinx" -a -E -b html . ./docs/html -w ./docs/_sphinx-warnings.txt -j "$JOBS" "$@" diff --git a/docs/changelog.txt b/docs/changelog.txt index d3ed3e51c..dfda78c0b 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -1,16 +1,10 @@ === Scroll down for changes ===[[[ -===help +The text below is included in docs/Documentation.rst - see that file for more details on the changelog setup. +This is kept in this file as a quick syntax reference. -Entries in docs/NEWS.rst and docs/NEWS-dev.rst are generated from -docs/changelog.txt. NEWS.rst groups entries by stable releases, and NEWS-dev.rst -groups them by all releases (stable and development). For example, an entry -listed under "0.44.05-alpha1" in changelog.txt will be listed under that in -NEWS-dev.rst as well, but under "0.44.05-r1" in NEWS.rst (assuming that is the -closest stable release after 0.44.05-alpha1). An entry listed under a stable -release in changelog.txt will be listed under that release in both NEWS.rst and -NEWS-dev.rst. +===help changelog.txt uses a syntax similar to RST, with a few special sequences: @@ -39,6 +33,19 @@ changelog.txt uses a syntax similar to RST, with a few special sequences: # Future +## Fixes +- Fixed a segfault when attempting to start a headless session with a graphical PRINT_MODE setting +- Fixed an issue with the macOS launcher failing to un-quarantine some files +- `labormanager`: fixed handling of new jobs in 0.47 +- Fixed ``Units::isEggLayer``, ``Units::isGrazer``, ``Units::isMilkable``, ``Units::isTrainableHunting``, ``Units::isTrainableWar``, and ``Units::isTamable`` ignoring the unit's caste +- `RemoteFortressReader`: fixed a couple crashes that could result from decoding invalid enum items (``site_realization_building_type`` and ``improvement_type``) + +## Misc Improvements +- `confirm`: added a confirmation dialog for convicting dwarves of crimes + +## Ruby +- Updated ``item_find`` and ``building_find`` to use centralized logic that works on more screens + # 0.47.04-r1 ## Fixes diff --git a/docs/gen_changelog.py b/docs/gen_changelog.py index 8b1a6771e..3b4452f62 100644 --- a/docs/gen_changelog.py +++ b/docs/gen_changelog.py @@ -1,285 +1,5 @@ -import collections -import copy -import itertools -import os -import sys +import os, sys +sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sphinx_extensions')) -CHANGELOG_PATHS = ( - 'docs/changelog.txt', - 'scripts/changelog.txt', - 'library/xml/changelog.txt', -) - -CHANGELOG_SECTIONS = [ - 'New Plugins', - 'New Scripts', - 'New Tweaks', - 'New Features', - 'New Internal Commands', - 'Fixes', - 'Misc Improvements', - 'Removed', - 'API', - 'Internals', - 'Lua', - 'Ruby', - 'Structures', -] - -REPLACEMENTS = { - '`search`': '`search-plugin`', -} - -def to_title_case(word): - if word == word.upper(): - # Preserve acronyms - return word - return word[0].upper() + word[1:].lower() - -def find_all_indices(string, substr): - start = 0 - while True: - i = string.find(substr, start) - if i == -1: - return - yield i - start = i + 1 - -def replace_text(string, replacements): - for old_text, new_text in replacements.items(): - new_string = '' - new_string_end = 0 # number of characters from string in new_string - for i in find_all_indices(string, old_text): - if i > 0 and string[i - 1] == '!': - # exempt if preceded by '!' - new_string += string[new_string_end:i - 1] - new_string += old_text - else: - # copy until this occurrence - new_string += string[new_string_end:i] - new_string += new_text - new_string_end = i + len(old_text) - new_string += string[new_string_end:] - string = new_string - return string - -class ChangelogEntry(object): - def __init__(self, text, section, stable_version, dev_version): - text = text.lstrip('- ') - # normalize section to title case - self.section = ' '.join(map(to_title_case, section.strip().split())) - self.stable_version = stable_version - self.dev_version = dev_version - self.dev_only = text.startswith('@') - text = text.lstrip('@ ') - self.children = [] - - split_index = text.find(': ') - if split_index != -1: - self.feature, description = text[:split_index], text[split_index+1:] - if description.strip(): - self.children.insert(0, description.strip()) - else: - self.feature = text - self.feature = self.feature.replace(':\\', ':').rstrip(':') - - self.sort_key = self.feature.upper() - - def __repr__(self): - return 'ChangelogEntry(%r, %r)' % (self.feature, self.children) - -def parse_changelog(): - entries = [] - - for fpath in CHANGELOG_PATHS: - if not os.path.isfile(fpath): - continue - with open(fpath) as f: - cur_stable = None - cur_dev = None - cur_section = None - last_entry = None - multiline = '' - for line_id, line in enumerate(f.readlines()): - line_id += 1 - - if multiline: - multiline += line - elif '[[[' in line: - multiline = line.replace('[[[', '') - - if ']]]' in multiline: - line = multiline.replace(']]]', '') - multiline = '' - elif multiline: - continue - - if not line.strip() or line.startswith('==='): - continue - - if line.startswith('##'): - cur_section = line.lstrip('#').strip() - elif line.startswith('#'): - cur_dev = line.lstrip('#').strip().lower() - if ('alpha' not in cur_dev and 'beta' not in cur_dev and - 'rc' not in cur_dev): - cur_stable = cur_dev - elif line.startswith('-'): - if not cur_stable or not cur_dev or not cur_section: - raise ValueError( - '%s:%i: Entry without section' % (fpath, line_id)) - last_entry = ChangelogEntry(line.strip(), cur_section, - cur_stable, cur_dev) - entries.append(last_entry) - elif line.lstrip().startswith('-'): - if not cur_stable or not cur_dev: - raise ValueError( - '%s:%i: Sub-entry without section' % (fpath, line_id)) - if not last_entry: - raise ValueError( - '%s:%i: Sub-entry without parent' % (fpath, line_id)) - last_entry.children.append(line.strip('- \n')) - else: - raise ValueError('Invalid line: ' + line) - - if not entries: - raise RuntimeError('No changelog files with contents found') - - return entries - -def consolidate_changelog(all_entries): - for sections in all_entries.values(): - for section, entries in sections.items(): - # sort() is stable, so reverse entries so that older entries for the - # same feature are on top - entries.reverse() - entries.sort(key=lambda entry: entry.sort_key) - new_entries = [] - for feature, group in itertools.groupby(entries, - lambda e: e.feature): - old_entries = list(group) - children = list(itertools.chain(*[entry.children - for entry in old_entries])) - new_entry = copy.deepcopy(old_entries[0]) - new_entry.children = children - new_entries.append(new_entry) - entries[:] = new_entries - - - -def print_changelog(versions, all_entries, path, replace=True, prefix=''): - # all_entries: version -> section -> entry - with open(path, 'w') as f: - def write(line): - if replace: - line = replace_text(line, REPLACEMENTS) - f.write(prefix + line + '\n') - for version in versions: - sections = all_entries[version] - if not sections: - continue - version = 'DFHack ' + version - write(version) - write('=' * len(version)) - write('') - for section in CHANGELOG_SECTIONS: - entries = sections[section] - if not entries: - continue - write(section) - write('-' * len(section)) - for entry in entries: - if len(entry.children) == 1: - write('- ' + entry.feature + ': ' + - entry.children[0].strip('- ')) - continue - elif entry.children: - write('- ' + entry.feature + ':') - write('') - for child in entry.children: - write(' - ' + child) - write('') - else: - write('- ' + entry.feature) - write('') - write('') - - -def generate_changelog(all=False): - entries = parse_changelog() - - # scan for unrecognized sections - for entry in entries: - if entry.section not in CHANGELOG_SECTIONS: - raise RuntimeWarning('Unknown section: ' + entry.section) - - # ordered versions - versions = ['future'] - # map versions to stable versions - stable_version_map = {} - # version -> section -> entry - stable_entries = collections.defaultdict(lambda: - collections.defaultdict(list)) - dev_entries = collections.defaultdict(lambda: - collections.defaultdict(list)) - - for entry in entries: - # build list of all versions - if entry.dev_version not in versions: - versions.append(entry.dev_version) - stable_version_map.setdefault(entry.dev_version, entry.stable_version) - - if not entry.dev_only: - # add non-dev-only entries to both changelogs - stable_entries[entry.stable_version][entry.section].append(entry) - dev_entries[entry.dev_version][entry.section].append(entry) - - consolidate_changelog(stable_entries) - consolidate_changelog(dev_entries) - - print_changelog(versions, stable_entries, 'docs/_auto/news.rst') - print_changelog(versions, dev_entries, 'docs/_auto/news-dev.rst') - - if all: - for version in versions: - if version not in stable_version_map: - print('warn: skipping ' + version) - continue - if stable_version_map[version] == version: - version_entries = {version: stable_entries[version]} - else: - version_entries = {version: dev_entries[version]} - print_changelog([version], version_entries, - 'docs/_changelogs/%s-github.txt' % version, - replace=False) - print_changelog([version], version_entries, - 'docs/_changelogs/%s-reddit.txt' % version, - replace=False, - prefix='> ') - - return entries - -if __name__ == '__main__': - import argparse - parser = argparse.ArgumentParser() - parser.add_argument('-a', '--all', action='store_true', - help='Print changelogs for all versions to docs/_changelogs') - parser.add_argument('-c', '--check', action='store_true', - help='Check that all entries are printed') - args = parser.parse_args() - - os.chdir(os.path.abspath(os.path.dirname(__file__))) - os.chdir('..') - entries = generate_changelog(all=args.all) - - if args.check: - with open('docs/_auto/news.rst') as f: - content_stable = f.read() - with open('docs/_auto/news-dev.rst') as f: - content_dev = f.read() - for entry in entries: - for description in entry.children: - if not entry.dev_only and description not in content_stable: - print('stable missing: ' + description) - if description not in content_dev: - print('dev missing: ' + description) +from dfhack.changelog import cli_entrypoint +cli_entrypoint() diff --git a/docs/index-about.rst b/docs/index-about.rst new file mode 100644 index 000000000..3dba1e5f8 --- /dev/null +++ b/docs/index-about.rst @@ -0,0 +1,13 @@ +============ +About DFHack +============ + +These pages contain information about the general DFHack project. + +.. toctree:: + :maxdepth: 1 + + /docs/NEWS + /docs/Authors + /LICENSE + /docs/Removed diff --git a/docs/index-dev.rst b/docs/index-dev.rst new file mode 100644 index 000000000..617c0c0d7 --- /dev/null +++ b/docs/index-dev.rst @@ -0,0 +1,19 @@ +======================== +DFHack Development Guide +======================== + +These are pages relevant to people developing for DFHack. + +.. toctree:: + :maxdepth: 1 + + /docs/Dev-intro + /docs/Contributing + /docs/Compile + /docs/NEWS-dev + /docs/Lua API + /docs/Documentation + /docs/Structures-intro + /docs/Memory-research + /docs/Binpatches + diff --git a/docs/sphinx_extensions/dfhack/__init__.py b/docs/sphinx_extensions/dfhack/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/docs/sphinx_extensions/dfhack/changelog.py b/docs/sphinx_extensions/dfhack/changelog.py new file mode 100644 index 000000000..6f444d36f --- /dev/null +++ b/docs/sphinx_extensions/dfhack/changelog.py @@ -0,0 +1,307 @@ +import collections +import copy +import itertools +import os +import sys + +from sphinx.errors import ExtensionError, SphinxError, SphinxWarning + +from dfhack.util import DFHACK_ROOT, DOCS_ROOT + +CHANGELOG_PATHS = ( + 'docs/changelog.txt', + 'scripts/changelog.txt', + 'library/xml/changelog.txt', +) +CHANGELOG_PATHS = (os.path.join(DFHACK_ROOT, p) for p in CHANGELOG_PATHS) + +CHANGELOG_SECTIONS = [ + 'New Plugins', + 'New Scripts', + 'New Tweaks', + 'New Features', + 'New Internal Commands', + 'Fixes', + 'Misc Improvements', + 'Removed', + 'API', + 'Internals', + 'Lua', + 'Ruby', + 'Structures', +] + +REPLACEMENTS = { + '`search`': '`search-plugin`', +} + +def to_title_case(word): + if word == word.upper(): + # Preserve acronyms + return word + return word[0].upper() + word[1:].lower() + +def find_all_indices(string, substr): + start = 0 + while True: + i = string.find(substr, start) + if i == -1: + return + yield i + start = i + 1 + +def replace_text(string, replacements): + for old_text, new_text in replacements.items(): + new_string = '' + new_string_end = 0 # number of characters from string in new_string + for i in find_all_indices(string, old_text): + if i > 0 and string[i - 1] == '!': + # exempt if preceded by '!' + new_string += string[new_string_end:i - 1] + new_string += old_text + else: + # copy until this occurrence + new_string += string[new_string_end:i] + new_string += new_text + new_string_end = i + len(old_text) + new_string += string[new_string_end:] + string = new_string + return string + +class ChangelogEntry(object): + def __init__(self, text, section, stable_version, dev_version): + text = text.lstrip('- ') + # normalize section to title case + self.section = ' '.join(map(to_title_case, section.strip().split())) + self.stable_version = stable_version + self.dev_version = dev_version + self.dev_only = text.startswith('@') + text = text.lstrip('@ ') + self.children = [] + + split_index = text.find(': ') + if split_index != -1: + self.feature, description = text[:split_index], text[split_index+1:] + if description.strip(): + self.children.insert(0, description.strip()) + else: + self.feature = text + self.feature = self.feature.replace(':\\', ':').rstrip(':') + + self.sort_key = self.feature.upper() + + def __repr__(self): + return 'ChangelogEntry(%r, %r)' % (self.feature, self.children) + +def parse_changelog(): + entries = [] + + for fpath in CHANGELOG_PATHS: + if not os.path.isfile(fpath): + continue + with open(fpath) as f: + cur_stable = None + cur_dev = None + cur_section = None + last_entry = None + multiline = '' + for line_id, line in enumerate(f.readlines()): + line_id += 1 + + if multiline: + multiline += line + elif '[[[' in line: + multiline = line.replace('[[[', '') + + if ']]]' in multiline: + line = multiline.replace(']]]', '') + multiline = '' + elif multiline: + continue + + if not line.strip() or line.startswith('==='): + continue + + if line.startswith('##'): + cur_section = line.lstrip('#').strip() + elif line.startswith('#'): + cur_dev = line.lstrip('#').strip().lower() + if ('alpha' not in cur_dev and 'beta' not in cur_dev and + 'rc' not in cur_dev): + cur_stable = cur_dev + elif line.startswith('-'): + if not cur_stable or not cur_dev or not cur_section: + raise ValueError( + '%s:%i: Entry without section' % (fpath, line_id)) + last_entry = ChangelogEntry(line.strip(), cur_section, + cur_stable, cur_dev) + entries.append(last_entry) + elif line.lstrip().startswith('-'): + if not cur_stable or not cur_dev: + raise ValueError( + '%s:%i: Sub-entry without section' % (fpath, line_id)) + if not last_entry: + raise ValueError( + '%s:%i: Sub-entry without parent' % (fpath, line_id)) + last_entry.children.append(line.strip('- \n')) + else: + raise ValueError('Invalid line: ' + line) + + if not entries: + raise RuntimeError('No changelog files with contents found') + + return entries + +def consolidate_changelog(all_entries): + for sections in all_entries.values(): + for section, entries in sections.items(): + # sort() is stable, so reverse entries so that older entries for the + # same feature are on top + entries.reverse() + entries.sort(key=lambda entry: entry.sort_key) + new_entries = [] + for feature, group in itertools.groupby(entries, + lambda e: e.feature): + old_entries = list(group) + children = list(itertools.chain(*[entry.children + for entry in old_entries])) + new_entry = copy.deepcopy(old_entries[0]) + new_entry.children = children + new_entries.append(new_entry) + entries[:] = new_entries + + + +def print_changelog(versions, all_entries, path, replace=True, prefix=''): + # all_entries: version -> section -> entry + with open(path, 'w') as f: + def write(line): + if replace: + line = replace_text(line, REPLACEMENTS) + f.write(prefix + line + '\n') + for version in versions: + sections = all_entries[version] + if not sections: + continue + version = 'DFHack ' + version + write(version) + write('=' * len(version)) + write('') + for section in CHANGELOG_SECTIONS: + entries = sections[section] + if not entries: + continue + write(section) + write('-' * len(section)) + for entry in entries: + if len(entry.children) == 1: + write('- ' + entry.feature + ': ' + + entry.children[0].strip('- ')) + continue + elif entry.children: + write('- ' + entry.feature + ':') + write('') + for child in entry.children: + write(' - ' + child) + write('') + else: + write('- ' + entry.feature) + write('') + write('') + + +def generate_changelog(all=False): + entries = parse_changelog() + + # scan for unrecognized sections + for entry in entries: + if entry.section not in CHANGELOG_SECTIONS: + raise SphinxWarning('Unknown section: ' + entry.section) + + # ordered versions + versions = ['future'] + # map versions to stable versions + stable_version_map = {} + # version -> section -> entry + stable_entries = collections.defaultdict(lambda: + collections.defaultdict(list)) + dev_entries = collections.defaultdict(lambda: + collections.defaultdict(list)) + + for entry in entries: + # build list of all versions + if entry.dev_version not in versions: + versions.append(entry.dev_version) + stable_version_map.setdefault(entry.dev_version, entry.stable_version) + + if not entry.dev_only: + # add non-dev-only entries to both changelogs + stable_entries[entry.stable_version][entry.section].append(entry) + dev_entries[entry.dev_version][entry.section].append(entry) + + consolidate_changelog(stable_entries) + consolidate_changelog(dev_entries) + + print_changelog(versions, stable_entries, os.path.join(DOCS_ROOT, '_auto/news.rst')) + print_changelog(versions, dev_entries, os.path.join(DOCS_ROOT, '_auto/news-dev.rst')) + + if all: + for version in versions: + if version not in stable_version_map: + print('warn: skipping ' + version) + continue + if stable_version_map[version] == version: + version_entries = {version: stable_entries[version]} + else: + version_entries = {version: dev_entries[version]} + print_changelog([version], version_entries, + os.path.join(DOCS_ROOT, '_changelogs/%s-github.txt' % version), + replace=False) + print_changelog([version], version_entries, + os.path.join(DOCS_ROOT, '_changelogs/%s-reddit.txt' % version), + replace=False, + prefix='> ') + + return entries + +def cli_entrypoint(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('-a', '--all', action='store_true', + help='Print changelogs for all versions to docs/_changelogs') + parser.add_argument('-c', '--check', action='store_true', + help='Check that all entries are printed') + args = parser.parse_args() + + entries = generate_changelog(all=args.all) + + if args.check: + with open(os.path.join(DOCS_ROOT, '_auto/news.rst')) as f: + content_stable = f.read() + with open(os.path.join(DOCS_ROOT, '_auto/news-dev.rst')) as f: + content_dev = f.read() + for entry in entries: + for description in entry.children: + if not entry.dev_only and description not in content_stable: + print('stable missing: ' + description) + if description not in content_dev: + print('dev missing: ' + description) + + +def sphinx_entrypoint(app, config): + try: + generate_changelog() + except SphinxError: + raise + except Exception as e: + raise ExtensionError(str(e), e) + + +def setup(app): + app.connect('config-inited', sphinx_entrypoint) + + return { + 'version': '0.1', + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/docs/sphinx_extensions/dfhack/util.py b/docs/sphinx_extensions/dfhack/util.py new file mode 100644 index 000000000..71a432da4 --- /dev/null +++ b/docs/sphinx_extensions/dfhack/util.py @@ -0,0 +1,7 @@ +import os + +DFHACK_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +DOCS_ROOT = os.path.join(DFHACK_ROOT, 'docs') + +if not os.path.isdir(DOCS_ROOT): + raise ReferenceError('docs root not found: %s' % DOCS_ROOT) diff --git a/docs/styles/dfhack.css b/docs/styles/dfhack.css index d12348c31..daedd1120 100644 --- a/docs/styles/dfhack.css +++ b/docs/styles/dfhack.css @@ -1,6 +1,3 @@ -/* make sure to sync this with the base theme's css filename */ -@import url("alabaster.css"); - /* Keybinding CSS from the DF wiki; applies to :kbd:`` directives. * Use this directive for all keypresses, to make them look like keys. */ @@ -46,3 +43,16 @@ div.body p, div.body dd, div.body li, div.body blockquote { -webkit-hyphens: manual; hyphens: manual; } + +div.sphinxsidebar { + max-height: 100%; + overflow-y: auto; +} + +div.sphinxsidebar h3.logo-name a { + border-bottom: 1px dotted #999; +} + +div.body { + min-width: unset; +} diff --git a/docs/templates/about.html b/docs/templates/about.html new file mode 100644 index 000000000..48f98859f --- /dev/null +++ b/docs/templates/about.html @@ -0,0 +1,6 @@ + + +

diff --git a/index.rst b/index.rst index d4bd223c5..91c883b51 100644 --- a/index.rst +++ b/index.rst @@ -2,33 +2,23 @@ Welcome to DFHack's documentation! ################################## -Introduction -============ -DFHack is a Dwarf Fortress memory access library, distributed with -a wide variety of useful scripts and plugins. +DFHack is a memory editing library for `Dwarf Fortress `_ +that provides a unified, cross-platform environment where tools can be developed +to extend the game. The default distribution contains a variety of tools, including +bugfixes, interface improvements, automation tools, modding tools, and more. +There are also a variety of third-party tools available. -The project is currently hosted at https://www.github.com/DFHack/dfhack, -and can be downloaded from `the releases page -`_. +.. _quick-links: -All new releases are announced in :forums:`the bay12 forums thread <139553>`, -which is also a good place for discussion and questions. - -For users, it provides a significant suite of bugfixes and interface -enhancements by default, and more can be enabled. There are also many tools -(such as `workflow` or `autodump`) which can make life easier. -You can even add third-party scripts and plugins to do almost anything! - -For modders, DFHack makes many things possible. Custom reactions, new -interactions, magic creature abilities, and more can be set through `modtools` -and custom raws. Non-standard DFHack scripts and inits can be stored in the -raw directory, making raws or saves fully self-contained for distribution - -or for coexistence in a single DF install, even with incompatible components. - -For developers, DFHack unites the various ways tools access DF memory and -allows easier development of new tools. As an open-source project under -`various copyleft licences `, contributions are welcome. +Quick Links +=========== +* `Downloads `_ +* `Installation guide ` +* `Source code `_ + (**important:** read `compile` before attempting to build from source) +* `Bay 12 forums thread `_ +* `Bug tracker `_ User Manual =========== @@ -40,28 +30,5 @@ User Manual /docs/Core /docs/Plugins /docs/Scripts - -Other Contents -============== - -.. toctree:: - :maxdepth: 1 - - /docs/Authors - /LICENSE - /docs/NEWS - /docs/Scripts-removed - -For Developers -============== - -.. toctree:: - :maxdepth: 1 - - /Contributing - /docs/Compile - /docs/NEWS-dev - /docs/Lua API - /library/xml/SYNTAX - /library/xml/how-to-update - /docs/Binpatches + /docs/index-about + /docs/index-dev diff --git a/library/Core.cpp b/library/Core.cpp index c0a1155c7..a9fe9a69a 100644 --- a/library/Core.cpp +++ b/library/Core.cpp @@ -1692,14 +1692,21 @@ bool Core::Init() if (is_headless) { #ifdef LINUX_BUILD - auto endwin = (int(*)(void))dlsym(RTLD_DEFAULT, "endwin"); - if (endwin) + if (is_text_mode) { - endwin(); + auto endwin = (int(*)(void))dlsym(RTLD_DEFAULT, "endwin"); + if (endwin) + { + endwin(); + } + else + { + cerr << "endwin(): bind failed" << endl; + } } else { - cerr << "endwin(): bind failed" << endl; + cerr << "Headless mode requires PRINT_MODE:TEXT" << endl; } #else cerr << "Headless mode not supported on Windows" << endl; diff --git a/library/LuaTypes.cpp b/library/LuaTypes.cpp index ebc33b477..7f0711b20 100644 --- a/library/LuaTypes.cpp +++ b/library/LuaTypes.cpp @@ -1434,6 +1434,28 @@ void struct_identity::build_metatable(lua_State *state) SetPtrMethods(state, base+1, base+2); } +void other_vectors_identity::build_metatable(lua_State *state) +{ + int base = lua_gettop(state); + MakeFieldMetatable(state, this, meta_struct_index, meta_struct_newindex); + + EnableMetaField(state, base+2, "_enum"); + + LookupInTable(state, index_enum, &DFHACK_TYPEID_TABLE_TOKEN); + lua_setfield(state, base+1, "_enum"); + + auto keys = &index_enum->getKeys()[-index_enum->getFirstItem()]; + + for (int64_t i = 0; i < index_enum->getLastItem(); i++) + { + lua_getfield(state, base+2, keys[i]); + lua_rawseti(state, base+2, int(i)); + } + + SetStructMethod(state, base+1, base+2, meta_struct_field_reference, "_field"); + SetPtrMethods(state, base+1, base+2); +} + void global_identity::build_metatable(lua_State *state) { int base = lua_gettop(state); diff --git a/library/include/BitArray.h b/library/include/BitArray.h index cfb859bd0..1f558221a 100644 --- a/library/include/BitArray.h +++ b/library/include/BitArray.h @@ -568,4 +568,16 @@ namespace DFHack root->next = link; } }; + + template + struct DfOtherVectors + { + std::vector & operator[](O other_id) + { + CHECK_INVALID_ARGUMENT(size_t(other_id) < sizeof(T) / sizeof(std::vector)); + + auto vectors = reinterpret_cast *>(this); + return vectors[other_id]; + } + }; } diff --git a/library/include/DataDefs.h b/library/include/DataDefs.h index fd70b7ba8..f4eff8510 100644 --- a/library/include/DataDefs.h +++ b/library/include/DataDefs.h @@ -334,6 +334,23 @@ namespace DFHack virtual identity_type type() { return IDTYPE_UNION; } }; + class DFHACK_EXPORT other_vectors_identity : public struct_identity { + enum_identity *index_enum; + + public: + other_vectors_identity(size_t size, TAllocateFn alloc, + compound_identity *scope_parent, const char *dfhack_name, + struct_identity *parent, const struct_field_info *fields, + enum_identity *index_enum) : + struct_identity(size, alloc, scope_parent, dfhack_name, parent, fields), + index_enum(index_enum) + {} + + enum_identity *getIndexEnum() { return index_enum; } + + virtual void build_metatable(lua_State *state); + }; + #ifdef _MSC_VER typedef void *virtual_ptr; #else @@ -465,6 +482,7 @@ namespace df using DFHack::global_identity; using DFHack::struct_identity; using DFHack::union_identity; + using DFHack::other_vectors_identity; using DFHack::struct_field_info; using DFHack::struct_field_info_extra; using DFHack::bitfield_item_info; @@ -474,6 +492,7 @@ namespace df using DFHack::BitArray; using DFHack::DfArray; using DFHack::DfLinkedList; + using DFHack::DfOtherVectors; #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdelete-non-virtual-dtor" diff --git a/library/modules/Items.cpp b/library/modules/Items.cpp index 05f174ddf..0e02043dc 100644 --- a/library/modules/Items.cpp +++ b/library/modules/Items.cpp @@ -1477,9 +1477,9 @@ int32_t Items::createItem(df::item_type item_type, int16_t item_subtype, int16_t df::enums::game_type::game_type type = *df::global::gametype; prod->produce(unit, &out_products, &out_items, &in_reag, &in_items, 1, job_skill::NONE, - df::historical_entity::find(unit->civ_id), 0, + 0, df::historical_entity::find(unit->civ_id), ((type == df::enums::game_type::DWARF_MAIN) || (type == df::enums::game_type::DWARF_RECLAIM)) ? df::world_site::find(df::global::ui->site_id) : NULL, - 0); + NULL); if ( out_items.size() != 1 ) return -1; diff --git a/library/modules/Kitchen.cpp b/library/modules/Kitchen.cpp index 63372d564..fa4d1dea7 100644 --- a/library/modules/Kitchen.cpp +++ b/library/modules/Kitchen.cpp @@ -95,7 +95,7 @@ void Kitchen::denyPlantSeedCookery(t_materialIndex materialIndex) { ui->kitchen.item_types.push_back(item_type::SEEDS); ui->kitchen.item_subtypes.push_back(organicSubtype); - ui->kitchen.mat_types.push_back(type->material_defs.type_seed); + ui->kitchen.mat_types.push_back(type->material_defs.type[plant_material_def::seed]); ui->kitchen.mat_indices.push_back(materialIndex); ui->kitchen.exc_types.push_back(df::kitchen_exc_type::Cook); } @@ -103,7 +103,7 @@ void Kitchen::denyPlantSeedCookery(t_materialIndex materialIndex) { ui->kitchen.item_types.push_back(item_type::PLANT); ui->kitchen.item_subtypes.push_back(organicSubtype); - ui->kitchen.mat_types.push_back(type->material_defs.type_basic_mat); + ui->kitchen.mat_types.push_back(type->material_defs.type[plant_material_def::basic_mat]); ui->kitchen.mat_indices.push_back(materialIndex); ui->kitchen.exc_types.push_back(df::kitchen_exc_type::Cook); } diff --git a/library/modules/MapCache.cpp b/library/modules/MapCache.cpp index b61ba375e..ce3039ecf 100644 --- a/library/modules/MapCache.cpp +++ b/library/modules/MapCache.cpp @@ -899,8 +899,8 @@ t_matpair MapExtras::BlockInfo::getBaseMaterial(df::tiletype tt, df::coord2d pos { if (auto raw = df::plant_raw::find(plant->material)) { - rv.mat_type = raw->material_defs.type_basic_mat; - rv.mat_index = raw->material_defs.idx_basic_mat; + rv.mat_type = raw->material_defs.type[plant_material_def::basic_mat]; + rv.mat_index = raw->material_defs.idx[plant_material_def::basic_mat]; } } break; @@ -912,8 +912,8 @@ t_matpair MapExtras::BlockInfo::getBaseMaterial(df::tiletype tt, df::coord2d pos rv.mat_type = MaterialInfo::PLANT_BASE; if (auto raw = df::plant_raw::find(grass[x][y])) { - rv.mat_type = raw->material_defs.type_basic_mat; - rv.mat_index = raw->material_defs.idx_basic_mat; + rv.mat_type = raw->material_defs.type[plant_material_def::basic_mat]; + rv.mat_index = raw->material_defs.idx[plant_material_def::basic_mat]; } break; diff --git a/library/modules/Materials.cpp b/library/modules/Materials.cpp index 51d717d05..c341ea99e 100644 --- a/library/modules/Materials.cpp +++ b/library/modules/Materials.cpp @@ -261,7 +261,7 @@ bool MaterialInfo::findPlant(const std::string &token, const std::string &subtok // As a special exception, return the structural material with empty subtoken if (subtoken.empty()) - return decode(p->material_defs.type_basic_mat, p->material_defs.idx_basic_mat); + return decode(p->material_defs.type[plant_material_def::basic_mat], p->material_defs.idx[plant_material_def::basic_mat]); for (size_t j = 0; j < p->material.size(); j++) if (p->material[j]->id == subtoken) diff --git a/library/modules/Translation.cpp b/library/modules/Translation.cpp index e05154b44..282039c02 100644 --- a/library/modules/Translation.cpp +++ b/library/modules/Translation.cpp @@ -69,16 +69,7 @@ bool Translation::copyName(df::language_name * source, df::language_name * targe if (source == target) return true; - target->first_name = source->first_name; - target->nickname = source->nickname; - for (int i = 0; i < 7; i++) - { - target->words[i] = source->words[i]; - target->parts_of_speech[i] = source->parts_of_speech[i]; - } - target->language = source->language; - target->unknown = source->unknown; - target->has_name = source->has_name; + *target = *source; return true; } diff --git a/library/modules/Units.cpp b/library/modules/Units.cpp index c4b9e60fa..422060803 100644 --- a/library/modules/Units.cpp +++ b/library/modules/Units.cpp @@ -636,74 +636,50 @@ bool Units::isEggLayer(df::unit* unit) { CHECK_NULL_POINTER(unit); df::creature_raw *raw = world->raws.creatures.all[unit->race]; - for (auto caste = raw->caste.begin(); caste != raw->caste.end(); ++caste) - { - if ((*caste)->flags.is_set(caste_raw_flags::LAYS_EGGS) - || (*caste)->flags.is_set(caste_raw_flags::LAYS_UNUSUAL_EGGS)) - return true; - } - return false; + df::caste_raw *caste = raw->caste.at(unit->caste); + return caste->flags.is_set(caste_raw_flags::LAYS_EGGS) + || caste->flags.is_set(caste_raw_flags::LAYS_UNUSUAL_EGGS); } bool Units::isGrazer(df::unit* unit) { CHECK_NULL_POINTER(unit); df::creature_raw *raw = world->raws.creatures.all[unit->race]; - for (auto caste = raw->caste.begin(); caste != raw->caste.end(); ++caste) - { - if((*caste)->flags.is_set(caste_raw_flags::GRAZER)) - return true; - } - return false; + df::caste_raw *caste = raw->caste.at(unit->caste); + return caste->flags.is_set(caste_raw_flags::GRAZER); } bool Units::isMilkable(df::unit* unit) { CHECK_NULL_POINTER(unit); df::creature_raw *raw = world->raws.creatures.all[unit->race]; - for (auto caste = raw->caste.begin(); caste != raw->caste.end(); ++caste) - { - if((*caste)->flags.is_set(caste_raw_flags::MILKABLE)) - return true; - } - return false; + df::caste_raw *caste = raw->caste.at(unit->caste); + return caste->flags.is_set(caste_raw_flags::MILKABLE); } bool Units::isTrainableWar(df::unit* unit) { CHECK_NULL_POINTER(unit); df::creature_raw *raw = world->raws.creatures.all[unit->race]; - for (auto caste = raw->caste.begin(); caste != raw->caste.end(); ++caste) - { - if((*caste)->flags.is_set(caste_raw_flags::TRAINABLE_WAR)) - return true; - } - return false; + df::caste_raw *caste = raw->caste.at(unit->caste); + return caste->flags.is_set(caste_raw_flags::TRAINABLE_WAR); } bool Units::isTrainableHunting(df::unit* unit) { CHECK_NULL_POINTER(unit); df::creature_raw *raw = world->raws.creatures.all[unit->race]; - for (auto caste = raw->caste.begin(); caste != raw->caste.end(); ++caste) - { - if((*caste)->flags.is_set(caste_raw_flags::TRAINABLE_HUNTING)) - return true; - } - return false; + df::caste_raw *caste = raw->caste.at(unit->caste); + return caste->flags.is_set(caste_raw_flags::TRAINABLE_HUNTING); } bool Units::isTamable(df::unit* unit) { CHECK_NULL_POINTER(unit); df::creature_raw *raw = world->raws.creatures.all[unit->race]; - for (auto caste = raw->caste.begin(); caste != raw->caste.end(); ++caste) - { - if((*caste)->flags.is_set(caste_raw_flags::PET) || - (*caste)->flags.is_set(caste_raw_flags::PET_EXOTIC)) - return true; - } - return false; + df::caste_raw *caste = raw->caste.at(unit->caste); + return caste->flags.is_set(caste_raw_flags::PET) + || caste->flags.is_set(caste_raw_flags::PET_EXOTIC); } bool Units::isMale(df::unit* unit) diff --git a/library/xml b/library/xml index 0792fc020..b9028b0bb 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit 0792fc0202fb6a04bfdaa262bc36a3b14c8581e5 +Subproject commit b9028b0bb9ad40d3ad4dc3f10934bb61aa16629b diff --git a/package/darwin/dfhack b/package/darwin/dfhack index 1c2c7a175..5241f1a67 100755 --- a/package/darwin/dfhack +++ b/package/darwin/dfhack @@ -13,7 +13,7 @@ fi # attempt to remove quarantine flag: https://github.com/DFHack/dfhack/issues/1465 if ! test -f hack/quarantine-removed; then - find hack/ libs/ dwarfort.exe -name '*.dylib' -or -name '*.exe' -print0 | xargs -0 xattr -d com.apple.quarantine 2>&1 | grep -iv 'no such xattr' + find hack/ libs/ dwarfort.exe \( -name '*.dylib' -or -name '*.framework' -or -name '*.exe' \) -print0 | xargs -0 xattr -d com.apple.quarantine 2>&1 | grep -iv 'no such xattr' echo "quarantine flag removed on $(date); remove this file to re-run" > hack/quarantine-removed fi diff --git a/plugins/add-spatter.cpp b/plugins/add-spatter.cpp index 3d2d89dd1..459e28f25 100644 --- a/plugins/add-spatter.cpp +++ b/plugins/add-spatter.cpp @@ -249,7 +249,7 @@ struct product_hook : improvement_product { std::vector *in_reag, std::vector *in_items, int32_t quantity, df::job_skill skill, - df::historical_entity *entity, int32_t unk, df::world_site *site, void* unk2) + int32_t quality, df::historical_entity *entity, df::world_site *site, std::vector *unk2) ) { if (auto product = products[this]) { @@ -295,7 +295,7 @@ struct product_hook : improvement_product { return; } - INTERPOSE_NEXT(produce)(unit, out_products, out_items, in_reag, in_items, quantity, skill, entity, unk, site, unk2); + INTERPOSE_NEXT(produce)(unit, out_products, out_items, in_reag, in_items, quantity, skill, quality, entity, site, unk2); } }; diff --git a/plugins/autochop.cpp b/plugins/autochop.cpp index e7f3874d3..8f2225f72 100644 --- a/plugins/autochop.cpp +++ b/plugins/autochop.cpp @@ -258,7 +258,7 @@ static bool skip_plant(const df::plant * plant, bool *restricted) const df::plant_raw *plant_raw = df::plant_raw::find(plant->material); // Skip fruit trees if set. - if (skip.fruit_trees && plant_raw->material_defs.type_drink != -1) + if (skip.fruit_trees && plant_raw->material_defs.type[plant_material_def::drink] != -1) { if (restricted) *restricted = true; diff --git a/plugins/buildingplan-lib.cpp b/plugins/buildingplan-lib.cpp index 4e7cfd3fd..0c29284db 100644 --- a/plugins/buildingplan-lib.cpp +++ b/plugins/buildingplan-lib.cpp @@ -9,7 +9,6 @@ void debug(const string &msg) color_ostream_proxy out(Core::getInstance().getConsole()); out << "DEBUG (" << PLUGIN_VERSION << "): " << msg << endl; } -#define dbg Core::getInstance().getConsole() void enable_quickfort_fn(pair& pair) { pair.second = true; } diff --git a/plugins/confirm.cpp b/plugins/confirm.cpp index c616ac493..07aee4640 100644 --- a/plugins/confirm.cpp +++ b/plugins/confirm.cpp @@ -18,6 +18,7 @@ #include "df/general_ref.h" #include "df/general_ref_contained_in_itemst.h" #include "df/viewscreen_dwarfmodest.h" +#include "df/viewscreen_justicest.h" #include "df/viewscreen_layer_militaryst.h" #include "df/viewscreen_locationsst.h" #include "df/viewscreen_tradegoodsst.h" @@ -479,6 +480,7 @@ DEFINE_CONFIRMATION(uniform_delete, viewscreen_layer_militaryst); DEFINE_CONFIRMATION(note_delete, viewscreen_dwarfmodest); DEFINE_CONFIRMATION(route_delete, viewscreen_dwarfmodest); DEFINE_CONFIRMATION(location_retire, viewscreen_locationsst); +DEFINE_CONFIRMATION(convict, viewscreen_justicest); DFhackCExport command_result plugin_init (color_ostream &out, vector &commands) { diff --git a/plugins/createitem.cpp b/plugins/createitem.cpp index 939afebc2..4ef1c853c 100644 --- a/plugins/createitem.cpp +++ b/plugins/createitem.cpp @@ -86,8 +86,8 @@ bool makeItem (df::reaction_product_itemst *prod, df::unit *unit, bool second_it building = df::building::find(dest_building); prod->produce(unit, &out_products, &out_items, &in_reag, &in_items, 1, job_skill::NONE, - df::historical_entity::find(unit->civ_id), 0, - (World::isFortressMode()) ? df::world_site::find(ui->site_id) : NULL, 0); + 0, df::historical_entity::find(unit->civ_id), + (World::isFortressMode()) ? df::world_site::find(ui->site_id) : NULL, NULL); if (!out_items.size()) return false; // if we asked to make shoes and we got twice as many as we asked, then we're okay diff --git a/plugins/diggingInvaders/assignJob.cpp b/plugins/diggingInvaders/assignJob.cpp index b253e7833..7e3dfaf8c 100644 --- a/plugins/diggingInvaders/assignJob.cpp +++ b/plugins/diggingInvaders/assignJob.cpp @@ -256,8 +256,8 @@ int32_t assignJob(color_ostream& out, Edge firstImportantEdge, unordered_map in_reag; vector in_items; prod->produce(firstInvader, &out_products, &out_items, &in_reag, &in_items, 1, df::job_skill::NONE, - df::historical_entity::find(firstInvader->civ_id), 0, - df::world_site::find(df::global::ui->site_id), 0); + 0, df::historical_entity::find(firstInvader->civ_id), + df::world_site::find(df::global::ui->site_id), NULL); if ( out_items.size() != 1 ) { out.print("%s, %d: wrong size: %zu.\n", __FILE__, __LINE__, out_items.size()); diff --git a/plugins/eventful.cpp b/plugins/eventful.cpp index 5d8dfdb59..42cfbdf69 100644 --- a/plugins/eventful.cpp +++ b/plugins/eventful.cpp @@ -287,12 +287,12 @@ struct product_hook : item_product { std::vector *in_reag, std::vector *in_items, int32_t quantity, df::job_skill skill, - df::historical_entity *entity, int32_t unk, df::world_site *site, void* unk2) + int32_t quality, df::historical_entity *entity, df::world_site *site, std::vector *unk2) ) { color_ostream_proxy out(Core::getInstance().getConsole()); auto product = products[this]; if ( !product ) { - INTERPOSE_NEXT(produce)(unit, out_products, out_items, in_reag, in_items, quantity, skill, entity, unk, site, unk2); + INTERPOSE_NEXT(produce)(unit, out_products, out_items, in_reag, in_items, quantity, skill, quality, entity, site, unk2); return; } df::reaction* this_reaction=product->react; @@ -304,7 +304,7 @@ struct product_hook : item_product { size_t out_item_count = out_items->size(); - INTERPOSE_NEXT(produce)(unit, out_products, out_items, in_reag, in_items, quantity, skill, entity, unk, site, unk2); + INTERPOSE_NEXT(produce)(unit, out_products, out_items, in_reag, in_items, quantity, skill, quality, entity, site, unk2); if ( out_items->size() == out_item_count ) return; //if it produced something, call the scripts diff --git a/plugins/getplants.cpp b/plugins/getplants.cpp index d1ee79a04..b4b7a493b 100644 --- a/plugins/getplants.cpp +++ b/plugins/getplants.cpp @@ -70,7 +70,7 @@ enum class selectability { //selectability selectablePlant(color_ostream &out, const df::plant_raw *plant, bool farming) selectability selectablePlant(const df::plant_raw *plant, bool farming) { - const DFHack::MaterialInfo basic_mat = DFHack::MaterialInfo(plant->material_defs.type_basic_mat, plant->material_defs.idx_basic_mat); + const DFHack::MaterialInfo basic_mat = DFHack::MaterialInfo(plant->material_defs.type[plant_material_def::basic_mat], plant->material_defs.idx[plant_material_def::basic_mat]); bool outOfSeason = false; selectability result = selectability::Nonselectable; @@ -92,7 +92,7 @@ selectability selectablePlant(const df::plant_raw *plant, bool farming) return selectability::Grass; } - if (farming && plant->material_defs.type_seed == -1) + if (farming && plant->material_defs.type[plant_material_def::seed] == -1) { return selectability::Nonselectable; } @@ -163,8 +163,8 @@ selectability selectablePlant(const df::plant_raw *plant, bool farming) { for (size_t k = 0; growth_mat.material->reaction_product.material.mat_type.size(); k++) { - if (growth_mat.material->reaction_product.material.mat_type[k] == plant->material_defs.type_seed && - growth_mat.material->reaction_product.material.mat_index[k] == plant->material_defs.idx_seed) + if (growth_mat.material->reaction_product.material.mat_type[k] == plant->material_defs.type[plant_material_def::seed] && + growth_mat.material->reaction_product.material.mat_index[k] == plant->material_defs.idx[plant_material_def::seed]) { seedSource = true; break; @@ -193,7 +193,7 @@ selectability selectablePlant(const df::plant_raw *plant, bool farming) } /* else if (plant->growths[i]->behavior.bits.has_seed) // This code designates beans, etc. when DF doesn't, but plant gatherers still fail to collect anything, so it's useless: bug #0006940. { - const DFHack::MaterialInfo seed_mat = DFHack::MaterialInfo(plant->material_defs.type_seed, plant->material_defs.idx_seed); + const DFHack::MaterialInfo seed_mat = DFHack::MaterialInfo(plant->material_defs.type[plant_material_def::seed], plant->material_defs.idx[plant_material_def::seed]); if (seed_mat.material->flags.is_set(material_flags::EDIBLE_RAW) || seed_mat.material->flags.is_set(material_flags::EDIBLE_COOKED)) diff --git a/plugins/labormanager/joblabormapper.cpp b/plugins/labormanager/joblabormapper.cpp index 7c4f80fb7..7b4e77553 100644 --- a/plugins/labormanager/joblabormapper.cpp +++ b/plugins/labormanager/joblabormapper.cpp @@ -896,6 +896,10 @@ JobLaborMapper::JobLaborMapper() job_to_labor_table[df::job_type::PutItemOnDisplay] = jlf_const(df::unit_labor::HAUL_ITEM); job_to_labor_table[df::job_type::StoreItemInLocation] = jlf_no_labor; // StoreItemInLocation + + job_to_labor_table[df::job_type::unk_fake_no_job] = jlf_no_labor; // added for 47.04 - see #1561 + job_to_labor_table[df::job_type::InterrogateSubject] = jlf_no_labor; // added for 47.04 - see #1561 + job_to_labor_table[df::job_type::unk_fake_no_activity] = jlf_no_labor; // added for 47.04 - see #1561 }; df::unit_labor JobLaborMapper::find_job_labor(df::job* j) diff --git a/plugins/labormanager/labormanager.cpp b/plugins/labormanager/labormanager.cpp index e416dcf63..fad421104 100644 --- a/plugins/labormanager/labormanager.cpp +++ b/plugins/labormanager/labormanager.cpp @@ -385,6 +385,9 @@ static const dwarf_state dwarf_states[] = { BUSY /* MakeBracelet */, BUSY /* MakeGem */, BUSY /* PutItemOnDisplay */, + OTHER /* unk_fake_no_job */, + OTHER /* InterrogateSubject */, + OTHER /* unk_fake_no_activity */, }; struct labor_info diff --git a/plugins/lua/confirm.lua b/plugins/lua/confirm.lua index 4312ab4be..b8a815662 100644 --- a/plugins/lua/confirm.lua +++ b/plugins/lua/confirm.lua @@ -204,6 +204,21 @@ end location_retire.title = "Retire location" location_retire.message = "Are you sure you want to retire this location?" +convict = defconf('convict') +convict.title = "Confirm conviction" +function convict.intercept_key(key) + return key == keys.SELECT and + screen.cur_column == df.viewscreen_justicest.T_cur_column.ConvictChoices +end +function convict.get_message() + name = dfhack.TranslateName(screen.convict_choices[screen.cursor_right].name) + if name == "" then + name = "this creature" + end + return "Are you sure you want to convict " .. name .. "?\n" .. + "This action is irreversible." +end + -- End of confirmation definitions function check() diff --git a/plugins/manipulator.cpp b/plugins/manipulator.cpp index e9206dc0f..a013df335 100644 --- a/plugins/manipulator.cpp +++ b/plugins/manipulator.cpp @@ -2032,7 +2032,9 @@ void viewscreen_unitlaborsst::render() int x = 1, y = 3 + num_rows + 2; Screen::Pen white_pen(' ', 15, 0); - Screen::paintString(white_pen, x, y, (cur->unit && cur->unit->sex) ? "\x0b" : "\x0c"); + auto symbol = cur->unit ? ENUM_ATTR(pronoun_type, symbol, cur->unit->sex) : nullptr; + if (symbol) + Screen::paintString(white_pen, x, y, symbol); x += 2; Screen::paintString(white_pen, x, y, cur->transname); x += cur->transname.length(); diff --git a/plugins/proto/RemoteFortressReader.proto b/plugins/proto/RemoteFortressReader.proto index a54a0397f..8255f91cd 100644 --- a/plugins/proto/RemoteFortressReader.proto +++ b/plugins/proto/RemoteFortressReader.proto @@ -642,33 +642,6 @@ message WorldMap repeated RegionTile region_tiles = 25; } -enum SiteRealizationBuildingType -{ - cottage_plot = 0; - castle_wall = 1; - castle_tower = 2; - castle_courtyard = 3; - house = 4; - temple = 5; - tomb = 6; - shop_house = 7; - warehouse = 8; - market_square = 9; - pasture = 10; - waste = 11; - courtyard = 12; - well = 13; - vault = 14; - great_tower = 15; - trenches = 16; - tree_house = 17; - hillock_house = 18; - mead_hall = 19; - fortress_entrance = 20; - library = 21; - tavern = 22; -} - message SiteRealizationBuildingWall { optional int32 start_x = 1; @@ -702,7 +675,6 @@ message SiteRealizationBuildingTrenches message SiteRealizationBuilding { optional int32 id = 1; - optional SiteRealizationBuildingType type = 2; optional int32 min_x = 3; optional int32 min_y = 4; optional int32 max_x = 5; @@ -711,6 +683,7 @@ message SiteRealizationBuilding optional SiteRealizationBuildingWall wall_info = 8; optional SiteRealizationBuildingTower tower_info = 9; optional SiteRealizationBuildingTrenches trench_info = 10; + optional int32 type = 11; } message RegionTile @@ -990,30 +963,13 @@ message Language repeated ShapeDescriptior shapes = 1; } -enum ImprovementType -{ - ART_IMAGE = 0; - COVERED = 1; - RINGS_HANGING = 2; - BANDS = 3; - SPIKES = 4; - ITEMSPECIFIC = 5; - THREAD = 6; - CLOTH = 7; - SEWN_IMAGE = 8; - PAGES = 9; - ILLUSTRATION = 10; - INSTRUMENT_PIECE = 11; - WRITING = 12; -} - message ItemImprovement { optional MatPair material = 1; - optional ImprovementType type = 2; optional int32 shape = 3; optional int32 specific_type= 4; optional ArtImage image = 5; + optional int32 type = 6; } enum ArtImageElementType diff --git a/plugins/remotefortressreader/item_reader.cpp b/plugins/remotefortressreader/item_reader.cpp index 620e06310..56ec9019b 100644 --- a/plugins/remotefortressreader/item_reader.cpp +++ b/plugins/remotefortressreader/item_reader.cpp @@ -450,7 +450,7 @@ void CopyItem(RemoteFortressReader::Item * NetItem, df::item * DfItem) auto netImp = NetItem->add_improvements(); - netImp->set_type((ImprovementType)impType); + netImp->set_type(impType); auto mat = netImp->mutable_material(); mat->set_mat_type(improvement->mat_type); diff --git a/plugins/remotefortressreader/remotefortressreader.cpp b/plugins/remotefortressreader/remotefortressreader.cpp index e94c3f832..44c476357 100644 --- a/plugins/remotefortressreader/remotefortressreader.cpp +++ b/plugins/remotefortressreader/remotefortressreader.cpp @@ -1,5 +1,5 @@ #include "df_version_int.h" -#define RFR_VERSION "0.20.3" +#define RFR_VERSION "0.21.0" #include #include @@ -949,9 +949,6 @@ static command_result GetGrowthList(color_ostream &stream, const EmptyMessage *i void CopyBlock(df::map_block * DfBlock, RemoteFortressReader::MapBlock * NetBlock, MapExtras::MapCache * MC, DFCoord pos) { - NetBlock->set_map_x(DfBlock->map_pos.x); - NetBlock->set_map_y(DfBlock->map_pos.y); - NetBlock->set_map_z(DfBlock->map_pos.z); MapExtras::Block * block = MC->BlockAtTile(DfBlock->map_pos); @@ -1465,7 +1462,12 @@ static command_result GetBlockList(color_ostream &stream, const BlockRequest *in bool flows = block->flows.size() > 0; RemoteFortressReader::MapBlock *net_block = nullptr; if (tileChanged || desChanged || spatterChanged || firstBlock || itemsChanged || flows) + { net_block = out->add_map_blocks(); + net_block->set_map_x(block->map_pos.x); + net_block->set_map_y(block->map_pos.y); + net_block->set_map_z(block->map_pos.z); + } if (tileChanged) { CopyBlock(block, net_block, &MC, pos); @@ -2541,7 +2543,7 @@ static void CopyLocalMap(df::world_data * worldData, df::world_region_details* w out_building->set_id(in_building->id); #if DF_VERSION_INT > 34011 - out_building->set_type((SiteRealizationBuildingType)in_building->type); + out_building->set_type(in_building->type); #endif out_building->set_min_x(in_building->min_x - (site_x * 48)); out_building->set_min_y(in_building->min_y - (site_y * 48)); @@ -2685,7 +2687,7 @@ static command_result GetPartialCreatureRaws(color_ostream &stream, const ListRe send_caste->add_child_name(orig_caste->child_name[0]); send_caste->add_child_name(orig_caste->child_name[1]); - send_caste->set_gender(orig_caste->gender); + send_caste->set_gender(orig_caste->sex); for (size_t partIndex = 0; partIndex < orig_caste->body_info.body_parts.size(); partIndex++) { diff --git a/plugins/rendermax/renderer_light.cpp b/plugins/rendermax/renderer_light.cpp index dda928afa..d76889220 100644 --- a/plugins/rendermax/renderer_light.cpp +++ b/plugins/rendermax/renderer_light.cpp @@ -270,7 +270,7 @@ rgbf blend(const rgbf& a,const rgbf& b) void lightingEngineViewscreen::clear() { lightMap.assign(lightMap.size(),rgbf(1,1,1)); - tthread::lock_guard guard(myRenderer->dataMutex); + std::lock_guard guard{myRenderer->dataMutex}; if(lightMap.size()==myRenderer->lightGrid.size()) { std::swap(myRenderer->lightGrid,lightMap); @@ -299,7 +299,7 @@ void lightingEngineViewscreen::calculate() } void lightingEngineViewscreen::updateWindow() { - tthread::lock_guard guard(myRenderer->dataMutex); + std::lock_guard guard{myRenderer->dataMutex}; if(lightMap.size()!=myRenderer->lightGrid.size()) { reinit(); diff --git a/plugins/rendermax/renderer_light.hpp b/plugins/rendermax/renderer_light.hpp index b0fb8c5d5..4e779f3fa 100644 --- a/plugins/rendermax/renderer_light.hpp +++ b/plugins/rendermax/renderer_light.hpp @@ -1,11 +1,14 @@ -#ifndef RENDERER_LIGHT_INCLUDED -#define RENDERER_LIGHT_INCLUDED -#include "renderer_opengl.hpp" -#include "Types.h" -#include -#include +#pragma once + #include +#include +#include +#include #include + +#include "renderer_opengl.hpp" +#include "Types.h" + // we are not using boost so let's cheat: template inline void hash_combine(std::size_t & seed, const T & v) @@ -91,7 +94,7 @@ private: } void reinitLightGrid(int w,int h) { - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; lightGrid.resize(w*h,rgbf(1,1,1)); } void reinitLightGrid() @@ -100,7 +103,7 @@ private: } public: - tthread::fast_mutex dataMutex; + std::mutex dataMutex; std::vector lightGrid; renderer_light(renderer* parent):renderer_wrap(parent),light_adaptation(1) { @@ -108,12 +111,12 @@ public: } virtual void update_tile(int32_t x, int32_t y) { renderer_wrap::update_tile(x,y); - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; colorizeTile(x,y); }; virtual void update_all() { renderer_wrap::update_all(); - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; for (int x = 0; x < df::global::gps->dimx; x++) for (int y = 0; y < df::global::gps->dimy; y++) colorizeTile(x,y); @@ -374,4 +377,3 @@ private: }; rgbf blend(const rgbf& a,const rgbf& b); rgbf blendMax(const rgbf& a,const rgbf& b); -#endif diff --git a/plugins/rendermax/renderer_opengl.hpp b/plugins/rendermax/renderer_opengl.hpp index 76a45918c..a0a920e69 100644 --- a/plugins/rendermax/renderer_opengl.hpp +++ b/plugins/rendermax/renderer_opengl.hpp @@ -1,9 +1,7 @@ //original file from https://github.com/Baughn/Dwarf-Fortress--libgraphics- -#ifndef RENDERER_OPENGL_INCLUDED -#define RENDERER_OPENGL_INCLUDED +#pragma once #include "tinythread.h" -#include "fast_mutex.h" #include "Core.h" #include @@ -15,6 +13,7 @@ #include "df/graphic.h" #include #include +#include using df::renderer; using df::init; @@ -281,7 +280,7 @@ private: } void reinitLightGrid(int w,int h) { - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; lightGrid.resize(w*h); } void reinitLightGrid() @@ -289,7 +288,7 @@ private: reinitLightGrid(df::global::gps->dimy,df::global::gps->dimx); } public: - tthread::fast_mutex dataMutex; + std::mutex dataMutex; std::vector lightGrid; renderer_test(renderer* parent):renderer_wrap(parent) { @@ -297,14 +296,14 @@ public: } virtual void update_tile(int32_t x, int32_t y) { renderer_wrap::update_tile(x,y); - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; colorizeTile(x,y); //some sort of mutex or sth? //and then map read }; virtual void update_all() { renderer_wrap::update_all(); - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; for (int x = 0; x < df::global::gps->dimx; x++) for (int y = 0; y < df::global::gps->dimy; y++) colorizeTile(x,y); @@ -366,7 +365,7 @@ private: } void reinitGrids(int w,int h) { - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; foreOffset.resize(w*h); foreMult.resize(w*h); backOffset.resize(w*h); @@ -377,7 +376,7 @@ private: reinitGrids(df::global::gps->dimy,df::global::gps->dimx); } public: - tthread::fast_mutex dataMutex; + std::mutex dataMutex; std::vector foreOffset,foreMult; std::vector backOffset,backMult; inline int xyToTile(int x, int y) @@ -390,14 +389,14 @@ public: } virtual void update_tile(int32_t x, int32_t y) { renderer_wrap::update_tile(x,y); - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; overwriteTile(x,y); //some sort of mutex or sth? //and then map read }; virtual void update_all() { renderer_wrap::update_all(); - tthread::lock_guard guard(dataMutex); + std::lock_guard guard{dataMutex}; for (int x = 0; x < df::global::gps->dimx; x++) for (int y = 0; y < df::global::gps->dimy; y++) overwriteTile(x,y); @@ -414,4 +413,3 @@ public: reinitGrids(w,h); } }; -#endif \ No newline at end of file diff --git a/plugins/rendermax/rendermax.cpp b/plugins/rendermax/rendermax.cpp index be8be07ba..17fd6fa95 100644 --- a/plugins/rendermax/rendermax.cpp +++ b/plugins/rendermax/rendermax.cpp @@ -1,27 +1,23 @@ -#include +#include +#include #include +#include -#include - -#include - -#include "Core.h" #include "Console.h" +#include "Core.h" #include "Export.h" +#include "LuaTools.h" #include "PluginManager.h" +#include "VTableInterpose.h" -#include -#include "df/renderer.h" #include "df/enabler.h" +#include "df/renderer.h" +#include "df/viewscreen_dungeonmodest.h" +#include "df/viewscreen_dwarfmodest.h" #include "renderer_opengl.hpp" #include "renderer_light.hpp" -#include "df/viewscreen_dwarfmodest.h" -#include "df/viewscreen_dungeonmodest.h" - -#include - using df::viewscreen_dungeonmodest; using df::viewscreen_dwarfmodest; @@ -367,7 +363,7 @@ static command_result rendermax(color_ostream &out, vector & parameters cur=blue; renderer_test* r=reinterpret_cast(enabler->renderer); - tthread::lock_guard guard(r->dataMutex); + std::lock_guard guard{r->dataMutex}; int h=gps->dimy; int w=gps->dimx; int cx=w/2; diff --git a/plugins/ruby/building.rb b/plugins/ruby/building.rb index 7a85e9314..dae1d455b 100644 --- a/plugins/ruby/building.rb +++ b/plugins/ruby/building.rb @@ -2,16 +2,7 @@ module DFHack class << self def building_find(what=:selected, y=nil, z=nil) if what == :selected - case ui.main.mode - when :LookAround - k = ui_look_list.items[ui_look_cursor] - k.building if k.type == :Building - when :BuildingItems, :QueryBuilding - world.selected_building - when :Zones, :ZonesPenInfo, :ZonesPitInfo, :ZonesHospitalInfo - ui_sidebar_menus.zone.selected - end - + return world.buildings.all.binsearch(df.get_selected_building_id) elsif what.kind_of?(Integer) # search by building.id return world.buildings.all.binsearch(what) if not z diff --git a/plugins/ruby/codegen.pl b/plugins/ruby/codegen.pl index 2a12a1bea..535a49694 100755 --- a/plugins/ruby/codegen.pl +++ b/plugins/ruby/codegen.pl @@ -240,6 +240,12 @@ sub render_global_class { } my $rbparent = ($parent ? rb_ucase($parent) : 'MemHack::Compound'); + my $ienum; + if (($type->getAttribute('ld:subtype') or '') eq 'df-other-vectors-type') + { + $rbparent = 'MemHack::OtherVectors'; + $ienum = rb_ucase($type->getAttribute('index-enum')); + } push @lines_rb, "class $rbname < $rbparent"; indent_rb { my $sz = sizeof($type); @@ -249,6 +255,8 @@ sub render_global_class { push @lines_rb, "rtti_classname :$rtti_name\n" if $rtti_name; + push @lines_rb, "ienum $ienum\n" if $ienum; + render_struct_fields($type); my $vms = $type->findnodes('child::virtual-methods')->[0]; diff --git a/plugins/ruby/item.rb b/plugins/ruby/item.rb index 394e545c6..59477c382 100644 --- a/plugins/ruby/item.rb +++ b/plugins/ruby/item.rb @@ -4,41 +4,7 @@ module DFHack # arg similar to unit.rb/unit_find; no arg = 'k' menu def item_find(what=:selected, y=nil, z=nil) if what == :selected - case curview._rtti_classname - when :viewscreen_itemst - if ref = curview.entry_ref[curview.cursor_pos] - ref.item_tg if ref.kind_of?(GeneralRefItem) - else - # not a container - curview.item - end - when :viewscreen_storesst # z/stocks - if curview.in_group_mode == 0 and curview.in_right_list == 1 - curview.items[curview.item_cursor] - end - else - case ui.main.mode - when :LookAround - k = ui_look_list.items[ui_look_cursor] - case k.type - when :Item - k.item - when :Building - # hilight a constructed bed/coffer - mats = k.building.contained_items.find_all { |i| i.use_mode == 2 } - mats[0].item if mats.length == 1 - end - when :BuildingItems - bld = world.selected_building - bld.contained_items[ui_building_item_cursor].item if bld - when :ViewUnits - u = world.units.active[ui_selected_unit] - u.inventory[ui_look_cursor].item if u and u.pos.z == cursor.z and - ui_unit_view_mode.value == :Inventory and u.inventory[ui_look_cursor] - else - ui.follow_item_tg if ui.follow_item != -1 - end - end + return world.items.all.binsearch(df.get_selected_item_id) elsif what.kind_of?(Integer) # search by id return world.items.all.binsearch(what) if not z diff --git a/plugins/ruby/ruby-autogen-defs.rb b/plugins/ruby/ruby-autogen-defs.rb index 10727894d..c72ffeb2b 100644 --- a/plugins/ruby/ruby-autogen-defs.rb +++ b/plugins/ruby/ruby-autogen-defs.rb @@ -205,6 +205,22 @@ module DFHack end end + class OtherVectors < Compound + class << self + attr_accessor :_enum + def ienum(enum) + @_enum = enum + end + end + + def [](i) + self.send(self.class._enum.sym(i)) + end + def []=(i, v) + self.send((self.class._enum.sym(i).to_s + "=").to_sym, v) + end + end + class Enum # number -> symbol def self.enum diff --git a/plugins/ruby/ruby.cpp b/plugins/ruby/ruby.cpp index 240a67cb2..0209806d1 100644 --- a/plugins/ruby/ruby.cpp +++ b/plugins/ruby/ruby.cpp @@ -9,6 +9,8 @@ #include "modules/Gui.h" #include "df/global_objects.h" +#include "df/building.h" +#include "df/item.h" #include "df/unit.h" #include "tinythread.h" @@ -630,6 +632,18 @@ static VALUE rb_dfget_vtable_ptr(VALUE self, VALUE objptr) return rb_uint2inum(*(uintptr_t*)rb_num2ulong(objptr)); } +static VALUE rb_dfget_selected_building_id(VALUE self) +{ + df::building *b = Gui::getAnyBuilding(Core::getTopViewscreen()); + return rb_int2inum(b ? b->id : -1); +} + +static VALUE rb_dfget_selected_item_id(VALUE self) +{ + df::item *i = Gui::getAnyItem(Core::getTopViewscreen()); + return rb_int2inum(i ? i->id : -1); +} + static VALUE rb_dfget_selected_unit_id(VALUE self) { df::unit *u = Gui::getAnyUnit(Core::getTopViewscreen()); @@ -1147,6 +1161,8 @@ static void ruby_bind_dfhack(void) { rb_define_singleton_method(rb_cDFHack, "get_vtable", RUBY_METHOD_FUNC(rb_dfget_vtable), 1); rb_define_singleton_method(rb_cDFHack, "get_rtti_classname", RUBY_METHOD_FUNC(rb_dfget_rtti_classname), 1); rb_define_singleton_method(rb_cDFHack, "get_vtable_ptr", RUBY_METHOD_FUNC(rb_dfget_vtable_ptr), 1); + rb_define_singleton_method(rb_cDFHack, "get_selected_building_id", RUBY_METHOD_FUNC(rb_dfget_selected_building_id), 0); + rb_define_singleton_method(rb_cDFHack, "get_selected_item_id", RUBY_METHOD_FUNC(rb_dfget_selected_item_id), 0); rb_define_singleton_method(rb_cDFHack, "get_selected_unit_id", RUBY_METHOD_FUNC(rb_dfget_selected_unit_id), 0); rb_define_singleton_method(rb_cDFHack, "dfhack_run", RUBY_METHOD_FUNC(rb_dfhack_run), 1); rb_define_singleton_method(rb_cDFHack, "print_str", RUBY_METHOD_FUNC(rb_dfprint_str), 1); diff --git a/plugins/stonesense b/plugins/stonesense index b9fc28836..bdab71c99 160000 --- a/plugins/stonesense +++ b/plugins/stonesense @@ -1 +1 @@ -Subproject commit b9fc28836f34f7ce1be64de94afd90184a341c7d +Subproject commit bdab71c99a0a7cc268ca517a0cd3f0a5fb41042a diff --git a/plugins/strangemood.cpp b/plugins/strangemood.cpp index e8a1046df..b1b48f1a5 100644 --- a/plugins/strangemood.cpp +++ b/plugins/strangemood.cpp @@ -153,7 +153,7 @@ void selectWord (const df::language_word_table &table, int32_t &word, df::part_o } } -void generateName(df::language_name &output, int language, int mode, const df::language_word_table &table1, const df::language_word_table &table2) +void generateName(df::language_name &output, int language, df::language_name_type mode, const df::language_word_table &table1, const df::language_word_table &table2) { for (int i = 0; i < 100; i++) { @@ -162,7 +162,7 @@ void generateName(df::language_name &output, int language, int mode, const df::l output = df::language_name(); if (language == -1) language = rng.df_trandom(world->raws.language.translations.size()); - output.unknown = mode; + output.type = mode; output.language = language; } output.has_name = 1; @@ -171,7 +171,9 @@ void generateName(df::language_name &output, int language, int mode, const df::l int r, r2, r3; switch (mode) { - case 0: case 9: case 10: + case language_name_type::Figure: + case language_name_type::FigureNoFirst: + case language_name_type::FigureFirstOnly: if (mode != 9) { int32_t word; df::part_of_speech part; @@ -182,7 +184,8 @@ void generateName(df::language_name &output, int language, int mode, const df::l } if (mode != 10) { - case 4: case 37: // this is not a typo + case language_name_type::Site: + case language_name_type::Unk37: // this is not a typo if (rng.df_trandom(2)) { selectWord(table2, output.words[0], output.parts_of_speech[0], 0); @@ -196,7 +199,9 @@ void generateName(df::language_name &output, int language, int mode, const df::l } break; - case 1: case 13: case 20: + case language_name_type::Artifact: + case language_name_type::Unk13: + case language_name_type::River: r = rng.df_trandom(3); if (r == 0 || r == 1) { @@ -213,7 +218,9 @@ void generateName(df::language_name &output, int language, int mode, const df::l } if (r == 1 || r == 2) { - case 3: case 8: case 11: // this is not a typo either + case language_name_type::Squad: + case language_name_type::LegendaryFigure: + case language_name_type::ArtImage: // this is not a typo either r2 = rng.df_trandom(2); if (r2) selectWord(table1, output.words[5], output.parts_of_speech[5], 2); @@ -256,9 +263,34 @@ void generateName(df::language_name &output, int language, int mode, const df::l output.parts_of_speech[5] = part_of_speech::NounPlural; break; - case 2: case 5: case 6: case 12: case 14: case 15: case 16: case 17: case 18: case 19: - case 21: case 22: case 23: case 24: case 25: case 26: case 27: case 28: case 29: case 30: - case 31: case 32: case 33: case 34: case 35: case 36: case 38: case 39: + case language_name_type::Civilization: + case language_name_type::World: + case language_name_type::Region: + case language_name_type::AdventuringGroup: + case language_name_type::SiteGovernment: + case language_name_type::Unk15: + case language_name_type::Vessel: + case language_name_type::MilitaryUnit: + case language_name_type::Religion: + case language_name_type::MountainPeak: + case language_name_type::Temple: + case language_name_type::Keep: + case language_name_type::MeadHall: + case language_name_type::Unk24: + case language_name_type::Unk25: + case language_name_type::Unk26: + case language_name_type::Market: + case language_name_type::Tavern: + case language_name_type::War: + case language_name_type::Battle: + case language_name_type::Siege: + case language_name_type::Road: + case language_name_type::Wall: + case language_name_type::Bridge: + case language_name_type::Tunnel: + case language_name_type::PretentiousEntityPosition: + case language_name_type::Tomb: + case language_name_type::MigratingGroup: selectWord(table1, output.words[5], output.parts_of_speech[5], 2); r3 = rng.df_trandom(3); if (rng.df_trandom(50)) @@ -280,7 +312,7 @@ void generateName(df::language_name &output, int language, int mode, const df::l selectWord(table2, output.words[4], output.parts_of_speech[4], 4); break; - case 7: + case language_name_type::Dungeon: r = rng.df_trandom(3); if (r == 0 || r == 1) { @@ -314,6 +346,9 @@ void generateName(df::language_name &output, int language, int mode, const df::l if (rng.df_trandom(100)) selectWord(table2, output.words[4], output.parts_of_speech[4], 4); break; + default: + // not handled yet + break; } if (output.words[2] != -1 && output.words[3] != -1 && world->raws.language.words[output.words[3]]->adj_dist < world->raws.language.words[output.words[2]]->adj_dist) @@ -1316,10 +1351,10 @@ command_result df_strangemood (color_ostream &out, vector & parameters) // Generate the artifact's name if (type == mood_type::Fell || type == mood_type::Macabre) - generateName(unit->status.artifact_name, unit->name.language, 1, world->raws.language.word_table[0][2], world->raws.language.word_table[1][2]); + generateName(unit->status.artifact_name, unit->name.language, language_name_type::Artifact, world->raws.language.word_table[0][2], world->raws.language.word_table[1][2]); else { - generateName(unit->status.artifact_name, unit->name.language, 1, world->raws.language.word_table[0][1], world->raws.language.word_table[1][1]); + generateName(unit->status.artifact_name, unit->name.language, language_name_type::Artifact, world->raws.language.word_table[0][1], world->raws.language.word_table[1][1]); if (!rng.df_trandom(100)) unit->status.artifact_name = unit->name; } diff --git a/plugins/tweak/tweaks/tradereq-pet-gender.h b/plugins/tweak/tweaks/tradereq-pet-gender.h index ca786bef2..c62a56f67 100644 --- a/plugins/tweak/tweaks/tradereq-pet-gender.h +++ b/plugins/tweak/tweaks/tradereq-pet-gender.h @@ -20,7 +20,7 @@ struct pet_gender_hook : df::viewscreen_topicmeeting_takerequestsst { vector& castes = entity->resources.animals.pet_castes; for (int i = (good_idx / 17) * 17, y = 4; i < (good_idx / 17) * 17 + 17 && size_t(i) < races.size(); i++, y++) { int x = 30 + 1 + world->raws.creatures.all[races[i]]->caste[castes[i]]->caste_name[0].size(); - bool male = (bool)world->raws.creatures.all[races[i]]->caste[castes[i]]->gender; + bool male = world->raws.creatures.all[races[i]]->caste[castes[i]]->sex == pronoun_type::he; OutputString((i == good_idx) ? COLOR_WHITE : COLOR_GREY, x, y, male ? "\013" : "\014"); } diff --git a/scripts b/scripts index 2079b9fb6..e8de92efb 160000 --- a/scripts +++ b/scripts @@ -1 +1 @@ -Subproject commit 2079b9fb69b8b4db48aa35ec54a96f5cca7cc8ef +Subproject commit e8de92efb73d5ef4d0b52df000d60d3350f07a37 diff --git a/travis/all.py b/travis/all.py old mode 100644 new mode 100755 index 281911e7a..edd2eee18 --- a/travis/all.py +++ b/travis/all.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import argparse, os, sys, time parser = argparse.ArgumentParser() diff --git a/travis/authors-rst.py b/travis/authors-rst.py old mode 100644 new mode 100755 index 2d7274219..076f7f226 --- a/travis/authors-rst.py +++ b/travis/authors-rst.py @@ -1,6 +1,7 @@ +#!/usr/bin/env python3 """ Overly-complicated script to check formatting/sorting in Authors.rst """ -import re, sys +import os, re, sys def main(): success = [True] @@ -9,6 +10,8 @@ def main(): for k in kwargs: info += ' %s %s:' % (k, kwargs[k]) print('line %i:%s %s' % (line, info, msg)) + if os.environ.get('GITHUB_ACTIONS'): + print('::error file=docs/Authors.rst,line=%i::%s %s' % (line, info.lstrip(), msg)) success[0] = False with open('docs/Authors.rst', 'rb') as f: lines = list(map(lambda line: line.decode('utf8').replace('\n', ''), f.readlines())) diff --git a/travis/check-rpc.py b/travis/check-rpc.py old mode 100644 new mode 100755 index ecddafa9f..aba3e3811 --- a/travis/check-rpc.py +++ b/travis/check-rpc.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import glob import sys diff --git a/travis/download-df.sh b/travis/download-df.sh index 834aa4da7..49dcedea7 100755 --- a/travis/download-df.sh +++ b/travis/download-df.sh @@ -1,8 +1,9 @@ #!/bin/sh +set -e + tardest="df.tar.bz2" -which md5sum && alias md5=md5sum selfmd5=$(openssl md5 < "$0") echo $selfmd5 @@ -28,7 +29,20 @@ if [ ! -f receipt ]; then patch=$(echo "$DF_VERSION" | cut -d. -f3) url="http://www.bay12games.com/dwarves/df_${minor}_${patch}_linux.tar.bz2" echo Downloading - wget "$url" -O "$tardest" + while read url; do + echo "Attempting download: ${url}" + if wget -v "$url" -O "$tardest"; then + break + fi + done < range_end + 1: + if range_start == range_end: + s += ('%i, ' % range_end) + else: + s += ('%i-%i, ' % (range_start, range_end)) + range_start = range_end = line + if i == len(lines) - 1: + s += ('%i' % line) + else: + range_end = line + if i == len(lines) - 1: + s += ('%i-%i, ' % (range_start, range_end)) + return s.rstrip(' ').rstrip(',') + +class LinterError(Exception): + def __init__(self, message, lines, total_lines): + self.message = message + self.lines = lines + self.total_lines = total_lines + + def __str__(self): + return '%s: %s' % (self.message, format_lines(self.lines, self.total_lines)) + + def github_actions_workflow_command(self, filename): + first_line = self.lines[0] if self.lines else 1 + return '::error file=%s,line=%i::%s' % (filename, first_line, self) class Linter(object): ignore = False @@ -33,36 +71,12 @@ class Linter(object): if not self.check_line(line): failures.append(i + 1) if len(failures): - raise LinterError('%s: %s' % (self.msg, self.display_lines(failures, len(lines)))) + raise LinterError(self.msg, failures, len(lines)) def fix(self, lines): for i in range(len(lines)): lines[i] = self.fix_line(lines[i]) - def display_lines(self, lines, total): - if len(lines) == total - 1: - return 'entire file' - if not len(lines): - # should never happen - return 'nowhere' - if len(lines) == 1: - return 'line %i' % lines[0] - s = 'lines ' - range_start = range_end = lines[0] - for i, line in enumerate(lines): - if line > range_end + 1: - if range_start == range_end: - s += ('%i, ' % range_end) - else: - s += ('%i-%i, ' % (range_start, range_end)) - range_start = range_end = line - if i == len(lines) - 1: - s += ('%i' % line) - else: - range_end = line - if i == len(lines) - 1: - s += ('%i-%i, ' % (range_start, range_end)) - return s.rstrip(' ').rstrip(',') class NewlineLinter(Linter): msg = 'Contains DOS-style newlines' @@ -91,6 +105,7 @@ class TabLinter(Linter): linters = [cls() for cls in Linter.__subclasses__() if not cls.ignore] def main(): + is_github_actions = bool(os.environ.get('GITHUB_ACTIONS')) root_path = os.path.abspath(sys.argv[1] if len(sys.argv) > 1 else '.') if not os.path.exists(root_path): print('Nonexistent path: %s' % root_path) @@ -112,13 +127,22 @@ def main(): try: lines[i] = line.decode('utf-8') except UnicodeDecodeError: - error('%s:%i: Invalid UTF-8 (other errors will be ignored)' % (rel_path, i + 1)) + msg_params = (rel_path, i + 1, 'Invalid UTF-8 (other errors will be ignored)') + if is_github_actions: + error() + print('::error file=%s,line=%i::%s' % msg_params) + else: + error('%s:%i: %s' % msg_params) lines[i] = '' for linter in linters: try: linter.check(lines) except LinterError as e: - error('%s: %s' % (rel_path, e)) + if is_github_actions: + error() + print(e.github_actions_workflow_command(rel_path)) + else: + error('%s: %s' % (rel_path, e)) if fix: linter.fix(lines) contents = '\n'.join(lines) diff --git a/travis/run-tests.py b/travis/run-tests.py old mode 100644 new mode 100755 index 70ab6cf20..f4c173af2 --- a/travis/run-tests.py +++ b/travis/run-tests.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import argparse import enum import json diff --git a/travis/script-docs.py b/travis/script-docs.py old mode 100644 new mode 100755 index 33b2818c8..b2c8063bf --- a/travis/script-docs.py +++ b/travis/script-docs.py @@ -1,10 +1,10 @@ -from __future__ import print_function -from io import open +#!/usr/bin/env python3 import os from os.path import basename, dirname, join, splitext import sys SCRIPT_PATH = sys.argv[1] if len(sys.argv) > 1 else 'scripts' +IS_GITHUB_ACTIONS = bool(os.environ.get('GITHUB_ACTIONS')) def expected_cmd(path): """Get the command from the name of a script.""" @@ -19,42 +19,61 @@ def check_ls(fname, line): line = line.strip() comment = '--' if fname.endswith('.lua') else '#' if '[====[' in line or not line.startswith(comment): - print('Error: no leading comment in ' + fname) + print_error('missing leading comment (requred for `ls`)', fname) return 1 return 0 +def print_error(message, filename, line=None): + if not isinstance(line, int): + line = 1 + print('Error: %s:%i: %s' % (filename, line, message)) + if IS_GITHUB_ACTIONS: + print('::error file=%s,line=%i::%s' % (filename, line, message)) + + def check_file(fname): errors, doclines = 0, [] tok1, tok2 = ('=begin', '=end') if fname.endswith('.rb') else \ ('[====[', ']====]') + doc_start_line = None with open(fname, errors='ignore') as f: lines = f.readlines() + if not lines: + print_error('empty file', fname) + return 1 errors += check_ls(fname, lines[0]) - for l in lines: + for i, l in enumerate(lines): if doclines or l.strip().endswith(tok1): + if not doclines: + doc_start_line = i + 1 doclines.append(l.rstrip()) if l.startswith(tok2): break else: if doclines: - print('Error: docs start but not end: ' + fname) + print_error('docs start but do not end', fname, doc_start_line) else: - print('Error: no documentation in: ' + fname) + print_error('no documentation found', fname) return 1 if not doclines: - print('Error: missing or malformed documentation in: ' + fname) + print_error('missing or malformed documentation', fname) return 1 title, underline = [d for d in doclines if d and '=begin' not in d and '[====[' not in d][:2] - if underline != '=' * len(title): - print('Error: title/underline mismatch:', fname, title, underline) + title_line = doc_start_line + doclines.index(title) + expected_underline = '=' * len(title) + if underline != expected_underline: + print_error('title/underline mismatch: expected {!r}, got {!r}'.format( + expected_underline, underline), + fname, title_line + 1) errors += 1 if title != expected_cmd(fname): - print('Warning: expected script title {}, got {}'.format( - expected_cmd(fname), title)) + print_error('expected script title {!r}, got {!r}'.format( + expected_cmd(fname), title), + fname, title_line) errors += 1 return errors diff --git a/travis/script-syntax.py b/travis/script-syntax.py old mode 100644 new mode 100755 index b3a264f04..53891f762 --- a/travis/script-syntax.py +++ b/travis/script-syntax.py @@ -1,8 +1,26 @@ +#!/usr/bin/env python3 import argparse import os import subprocess import sys + +def print_stderr(stderr, args): + if not args.github_actions: + sys.stderr.write(stderr + '\n') + return + + for line in stderr.split('\n'): + parts = list(map(str.strip, line.split(':'))) + # e.g. luac prints "luac:" in front of messages, so find the first part + # containing the actual filename + for i in range(len(parts) - 1): + if parts[i].endswith('.' + args.ext) and parts[i + 1].isdigit(): + print('::error file=%s,line=%s::%s' % (parts[i], parts[i + 1], ':'.join(parts[i + 2:]))) + break + print(line) + + def main(args): root_path = os.path.abspath(args.path) cmd = args.cmd.split(' ') @@ -19,7 +37,13 @@ def main(args): continue full_path = os.path.join(cur, filename) try: - subprocess.check_output(cmd + [full_path]) + p = subprocess.Popen(cmd + [full_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + _, stderr = p.communicate() + stderr = stderr.decode('utf-8', errors='ignore') + if stderr: + print_stderr(stderr, args) + if p.returncode != 0: + err = True except subprocess.CalledProcessError: err = True except IOError: @@ -34,5 +58,7 @@ if __name__ == '__main__': parser.add_argument('--path', default='.', help='Root directory') parser.add_argument('--ext', help='Script extension', required=True) parser.add_argument('--cmd', help='Command', required=True) + parser.add_argument('--github-actions', action='store_true', + help='Enable GitHub Actions workflow command output') args = parser.parse_args() main(args)