Merge branch 'develop' into modding-guide

.github/workflows/build.yml

@@ -86,6 +86,7 @@ jobs:
           -DBUILD_TESTS:BOOL=ON \
           -DBUILD_DEV_PLUGINS:BOOL=${{ matrix.plugins == 'all' }} \
           -DBUILD_SIZECHECK:BOOL=${{ matrix.plugins == 'all' }} \
+          -DBUILD_SKELETON:BOOL=${{ matrix.plugins == 'all' }} \
           -DBUILD_STONESENSE:BOOL=${{ matrix.plugins == 'all' }} \
           -DBUILD_SUPPORTED:BOOL=1 \
           -DCMAKE_C_COMPILER_LAUNCHER=ccache \
@@ -100,7 +101,6 @@ jobs:
       run: |
         export TERM=dumb
         status=0
-        mv "$DF_FOLDER"/dfhack.init-example "$DF_FOLDER"/dfhack.init
         script -qe -c "python ci/run-tests.py --headless --keep-status \"$DF_FOLDER\"" || status=$((status + 1))
         python ci/check-rpc.py "$DF_FOLDER/dfhack-rpc.txt" || status=$((status + 2))
         mkdir -p artifacts

.pre-commit-config.yaml

@@ -20,11 +20,11 @@ repos:
       args: ['--fix=lf']
     - id: trailing-whitespace
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.16.0
+  rev: 0.17.1
   hooks:
     - id: check-github-workflows
 - repo: https://github.com/Lucas-C/pre-commit-hooks
-  rev: v1.2.0
+  rev: v1.3.0
   hooks:
     - id: forbid-tabs
       exclude_types:
@@ -41,4 +41,4 @@ repos:
       entry: python3 ci/authors-rst.py
       files: docs/Authors\.rst
       pass_filenames: false
-exclude: '^(depends/|data/examples/.*\.json$|.*\.diff$)'
+exclude: '^(depends/|data/.*\.json$|.*\.diff$)'

CMakeLists.txt

@@ -206,11 +206,9 @@ set(DFHACK_BUILD_ID "" CACHE STRING "Build ID (should be specified on command li
 if(UNIX)
     # put the lib into DF/hack
     set(DFHACK_LIBRARY_DESTINATION hack)
-    set(DFHACK_EGGY_DESTINATION libs)
 else()
     # windows is crap, therefore we can't do nice things with it. leave the libs on a nasty pile...
     set(DFHACK_LIBRARY_DESTINATION .)
-    set(DFHACK_EGGY_DESTINATION .)
 endif()
 
 # external tools will be installed here:

ci/run-tests.py

@@ -68,7 +68,16 @@ init_contents = change_setting(init_contents, 'FPS', 'YES')
 if args.headless:
     init_contents = change_setting(init_contents, 'PRINT_MODE', 'TEXT')
 
-test_init_file = 'dfhackzzz_test.init'  # Core sorts these alphabetically
+init_path = 'dfhack-config/init'
+if not os.path.isdir('hack/init'):
+    # we're on an old branch that still reads init files from the root dir
+    init_path = '.'
+try:
+    os.mkdir(init_path)
+except OSError as error:
+    # ignore already exists errors
+    pass
+test_init_file = os.path.join(init_path, 'dfhackzzz_test.init')  # Core sorts these alphabetically
 with open(test_init_file, 'w') as f:
     f.write('''
     devel/dump-rpc dfhack-rpc.txt

ci/script-docs.py

@@ -1,12 +1,13 @@
 #!/usr/bin/env python3
 import os
-from os.path import basename, dirname, join, splitext
+from os.path import basename, dirname, exists, join, splitext
 import sys
 
 SCRIPT_PATH = sys.argv[1] if len(sys.argv) > 1 else 'scripts'
+DOCS_PATH = join(SCRIPT_PATH, 'docs')
 IS_GITHUB_ACTIONS = bool(os.environ.get('GITHUB_ACTIONS'))
 
-def expected_cmd(path):
+def get_cmd(path):
     """Get the command from the name of a script."""
     dname, fname = basename(dirname(path)), splitext(basename(path))[0]
     if dname in ('devel', 'fix', 'gui', 'modtools'):
@@ -14,16 +15,6 @@ def expected_cmd(path):
     return fname
 
 
-def check_ls(fname, line):
-    """Check length & existence of leading comment for "ls" builtin command."""
-    line = line.strip()
-    comment = '--' if fname.endswith('.lua') else '#'
-    if '[====[' in line or not line.startswith(comment):
-        print_error('missing leading comment (requred for `ls`)', fname)
-        return 1
-    return 0
-
-
 def print_error(message, filename, line=None):
     if not isinstance(line, int):
         line = 1
@@ -32,48 +23,42 @@ def print_error(message, filename, line=None):
         print('::error file=%s,line=%i::%s' % (filename, line, message))
 
 
+def check_ls(docfile, lines):
+    """Check length & existence of first sentence for "ls" builtin command."""
+    # TODO
+    return 0
+
+
 def check_file(fname):
-    errors, doclines = 0, []
+    errors, doc_start_line = 0, None
-    tok1, tok2 = ('=begin', '=end') if fname.endswith('.rb') else \
+    docfile = join(DOCS_PATH, get_cmd(fname)+'.rst')
-        ('[====[', ']====]')
+    if not exists(docfile):
-    doc_start_line = None
+        print_error('missing documentation file: {!r}'.format(docfile), fname)
-    with open(fname, errors='ignore') as f:
+        return 1
+    with open(docfile, errors='ignore') as f:
         lines = f.readlines()
         if not lines:
-            print_error('empty file', fname)
+            print_error('empty documentation file', docfile)
             return 1
-        errors += check_ls(fname, lines[0])
         for i, l in enumerate(lines):
-            if doclines or l.strip().endswith(tok1):
+            l = l.strip()
-                if not doclines:
+            if l and not doc_start_line and doc_start_line != 0:
-                    doc_start_line = i + 1
+                doc_start_line = i
-                doclines.append(l.rstrip())
+            doc_end_line = i
-            if l.startswith(tok2):
+            lines[i] = l
-                break
-        else:
-            if doclines:
-                print_error('docs start but do not end', fname, doc_start_line)
-            else:
-                print_error('no documentation found', fname)
-            return 1
-
-    if not doclines:
-        print_error('missing or malformed documentation', fname)
-        return 1
 
-    title, underline = [d for d in doclines
+    errors += check_ls(docfile, lines)
-                        if d and '=begin' not in d and '[====[' not in d][:2]
+    title, underline = lines[doc_start_line:doc_start_line+2]
-    title_line = doc_start_line + doclines.index(title)
     expected_underline = '=' * len(title)
     if underline != expected_underline:
         print_error('title/underline mismatch: expected {!r}, got {!r}'.format(
                 expected_underline, underline),
-            fname, title_line + 1)
+            docfile, doc_start_line+1)
         errors += 1
-    if title != expected_cmd(fname):
+    if title != get_cmd(fname):
         print_error('expected script title {!r}, got {!r}'.format(
-                expected_cmd(fname), title),
+                get_cmd(fname), title),
-            fname, title_line)
+            docfile, doc_start_line)
         errors += 1
     return errors
 

conf.py

@@ -24,21 +24,21 @@ import sys
 
 
 # -- Support :dfhack-keybind:`command` ------------------------------------
-# this is a custom directive that pulls info from dfhack.init-example
+# this is a custom directive that pulls info from default keybindings
 
 from docutils import nodes
 from docutils.parsers.rst import roles
 
 sphinx_major_version = sphinx.version_info[0]
 
-def get_keybinds():
+def get_keybinds(root, files, keybindings):
-    """Get the implemented keybinds, and return a dict of
+    """Add keybindings in the specified files to the
-    {tool: [(full_command, keybinding, context), ...]}.
+    given keybindings dict.
     """
-    with open('dfhack.init-example') as f:
+    for file in files:
+        with open(os.path.join(root, file)) as f:
             lines = [l.replace('keybinding add', '').strip() for l in f.readlines()
                      if l.startswith('keybinding add')]
-    keybindings = dict()
         for k in lines:
             first, command = k.split(' ', 1)
             bind, context = (first.split('@') + [''])[:2]
@@ -47,9 +47,17 @@ def get_keybinds():
             tool = command.split(' ')[0].replace('"', '')
             keybindings[tool] = keybindings.get(tool, []) + [
                 (command, bind.split('-'), context)]
+
+def get_all_keybinds(root_dir):
+    """Get the implemented keybinds, and return a dict of
+    {tool: [(full_command, keybinding, context), ...]}.
+    """
+    keybindings = dict()
+    for root, _, files in os.walk(root_dir):
+        get_keybinds(root, files, keybindings)
     return keybindings
 
-KEYBINDS = get_keybinds()
+KEYBINDS = get_all_keybinds('data/init')
 
 
 # pylint:disable=unused-argument,dangerous-default-value,too-many-arguments
@@ -283,6 +291,7 @@ exclude_patterns = [
     'build*',
     'docs/_auto/news*',
     'docs/_changelogs/',
+    'scripts/docs/*',
     ]
 
 # The reST default role (used for this markup: `text`) to use for all

data/CMakeLists.txt

@@ -1,9 +1,21 @@
+install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/init/
+        DESTINATION "${DFHACK_DATA_DESTINATION}/init")
+
+install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/base_command_counts.json
+        DESTINATION "${DFHACK_DATA_DESTINATION}/data/base_command_counts.json")
+
 install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/quickfort/
         DESTINATION "${DFHACK_DATA_DESTINATION}/data/quickfort")
 
+install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/orders/
+        DESTINATION dfhack-config/orders/library)
+
 install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/examples/
         DESTINATION "${DFHACK_DATA_DESTINATION}/examples")
 
+install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/professions/
+        DESTINATION dfhack-config/professions/library)
+
 install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/blueprints/
         DESTINATION blueprints
         FILES_MATCHING PATTERN "*"

data/base_command_counts.json

@@ -0,0 +1,145 @@
+{
+	"manipulator": 75,
+	"autolabor": 59,
+	"reveal": 51,
+	"help": 50,
+	"ls": 50,
+	"die": 50,
+	"tags": 50,
+	"embark-assistant": 42,
+	"prospect": 37,
+	"autodump": 36,
+	"clean": 35,
+	"gui/workflow": 28,
+	"workflow": 28,
+	"exportlegends": 26,
+	"gui/autobutcher": 25,
+	"autobutcher": 25,
+	"digv": 24,
+	"fastdwarf": 22,
+	"autonestbox": 20,
+	"showmood": 19,
+	"gui/liquids": 18,
+	"liquids": 18,
+	"search": 18,
+	"gui/quickfort": 15,
+	"quickfort": 15,
+	"createitem": 14,
+	"stocks": 14,
+	"autofarm": 12,
+	"autochop": 12,
+	"tiletypes": 12,
+	"exterminate": 12,
+	"buildingplan": 12,
+	"quicksave": 11,
+	"gui/gm-editor": 11,
+	"cleanowned": 10,
+	"gui/autogems": 9,
+	"autogems": 9,
+	"stonesense": 9,
+	"gui/stockpiles": 8,
+	"stockpiles": 8,
+	"changevein": 8,
+	"gui/teleport": 7,
+	"teleport": 7,
+	"seedwatch": 6,
+	"automelt": 6,
+	"embark-tools": 6,
+	"cursecheck": 5,
+	"open-legends": 5,
+	"ban-cooking": 5,
+	"burial": 5,
+	"automaterial": 5,
+	"remove-stress": 5,
+	"gui/blueprint": 5,
+	"blueprint": 5,
+	"tailor": 4,
+	"startdwarf": 4,
+	"3dveins": 4,
+	"digcircle": 4,
+	"nestboxes": 3,
+	"deathcause": 3,
+	"list-agreements": 3,
+	"gui/room-list": 3,
+	"points": 3,
+	"region-pops": 3,
+	"gui/advfort": 3,
+	"unsuspend": 3,
+	"locate-ore": 3,
+	"changelayer": 3,
+	"source": 3,
+	"gui/gm-unit": 3,
+	"combine-drinks": 3,
+	"combine-plants": 3,
+	"deteriorate": 3,
+	"warn-starving": 3,
+	"gaydar": 2,
+	"gui/dfstatus": 2,
+	"gui/rename": 2,
+	"rename": 2,
+	"fix-ster": 2,
+	"job-material": 2,
+	"stockflow": 2,
+	"drain-aquifer": 2,
+	"full-heal": 2,
+	"spawnunit": 2,
+	"flashstep": 2,
+	"gui/family-affairs": 2,
+	"caravan": 2,
+	"mousequery": 2,
+	"tweak": 2,
+	"confirm": 2,
+	"autoclothing": 1,
+	"autounsuspend": 1,
+	"prioritize": 1,
+	"dwarfmonitor": 1,
+	"show-unit-syndromes": 1,
+	"troubleshoot-item": 1,
+	"gui/mechanisms": 1,
+	"gui/pathable": 1,
+	"hotkeys": 1,
+	"infiniteSky": 1,
+	"force": 1,
+	"hermit": 1,
+	"strangemood": 1,
+	"weather": 1,
+	"add-recipe": 1,
+	"autotrade": 1,
+	"zone": 1,
+	"autonick": 1,
+	"stripcaged": 1,
+	"unforbid": 1,
+	"workorder": 1,
+	"gui/mod-manager": 1,
+	"spotclean": 1,
+	"plant": 1,
+	"regrass": 1,
+	"dig-now": 1,
+	"build-now": 1,
+	"clear-webs": 1,
+	"gui/siege-engine": 1,
+	"assign-skills": 1,
+	"brainwash": 1,
+	"elevate-mental": 1,
+	"elevate-physical": 1,
+	"launch": 1,
+	"linger": 1,
+	"make-legendary": 1,
+	"rejuvenate": 1,
+	"resurrect-adv": 1,
+	"questport": 1,
+	"getplants": 1,
+	"gui/stamper": 1,
+	"tweak": 1,
+	"fixveins": 1,
+	"deramp": 1,
+	"fix/dead-units": 1,
+	"fix/fat-dwarves": 1,
+	"fix/loyaltycascade": 1,
+	"fix/retrieve-units": 1,
+	"tweak": 1,
+	"sort-items": 1,
+	"gui/color-schemes": 1,
+	"color-schemes": 1,
+	"season-palette": 1
+}

data/blueprints/library/dreamfort.csv

@@ -29,9 +29,9 @@
 "Dreamfort works best at an embark site that is flat and has at least one soil layer. New players should avoid embarks with aquifers if they are not prepared to deal with them. Bring picks for mining, an axe for woodcutting, and an anvil for a forge. Bring a few blocks to speed up initial workshop construction as well. That's all you really need, but see the example embark profile in the online spreadsheets for a more complete setup."
 ""
 "Other DFHack commands also work very well with Dreamfort, such as autofarm, autonestbox, prioritize, seedwatch, tailor, and, of course, buildingplan. An init file that gets everything configured for you is distributed with DFHack as hack/examples/init/onMapLoad_dreamfort.init."
-Put that file in your Dwarf Fortress directory -- the same directory that has dfhack.init.
+Put that file in your dfhack-config/init/ directory -- the same directory that has dfhack.init.
 ""
-"Also copy the files in hack/examples/orders/ to dfhack-config/orders/ and the files in hack/examples/professions/ to professions/. We'll be using these files later. See https://docs.dfhack.org/en/stable/docs/guides/examples-guide.html for more information, including suggestions on how many dwarves of each profession you are likely to need at each stage of fort maturity."
+"Also check out https://docs.dfhack.org/en/stable/docs/Plugins.html#professions for more information on the default labor professions that are distributed with DFHack, including suggestions on how many dwarves of each profession you are likely to need at each stage of fort maturity."
 ""
 "Once you have your starting surface workshops up and running, you might want to configure buildingplan (in its global settings, accessible from any building placement screen, e.g.: b-a-G) to only use blocks for constructions so it won't use your precious wood, boulders, and bars to build floors and walls. If you bring at least 7 blocks with you on embark, you can even set this in your onMapLoad.init file like this:"
 on-new-fortress buildingplan set boulders false; buildingplan set logs false
@@ -49,10 +49,7 @@ interactively."
 "Here is the recommended order for Dreamfort commands. You can copy/paste the command lines directly into the DFHack terminal, or, if you prefer, you can run the blueprints in the UI with gui/quickfort. See the walkthroughs (the ""help"" blueprints) for context and details. Also remember to read the messages the blueprints print out after you run them so you don't miss any important manual steps."
 ""
 -- Preparation (before you embark!) --
-Copy hack/examples/init/onMapLoad_dreamfort.init to your DF directory
+Copy hack/examples/init/onMapLoad_dreamfort.init to your dfhack-config/init directory inside your DF installation
-Copy the fort automation orders from hack/examples/orders/*.json to the dfhack-config/orders/ directory
-Optionally copy the premade profession definitions from hack/examples/professions/ to the professions/ directory
-Optionally copy the premade Dreamfort embark profile from the online spreadsheets to the data/init/embark_profiles.txt file
 ""
 -- Set settings and preload initial orders --
 quickfort run library/dreamfort.csv -n /setup,# Run before making any manual adjustments to settings! Run the /setup_help blueprint for details on what this blueprint does.
@@ -74,36 +71,36 @@ quickfort run library/dreamfort.csv -n /farming3,# Run when furniture has been p
 quickfort run library/dreamfort.csv -n /industry2,# Run when the industry level has been dug out.
 prioritize ConstructBuilding,# To get those workshops up and running ASAP. You may have to run this several times as the materials for the building construction jobs become ready.
 quickfort run library/dreamfort.csv -n /surface4,"# Run after the walls and floors are built on the surface. Even if /surface3 is finished before you run /industry2, though, wait until after /industry2 to run this blueprint so that surface walls, floors, and roofing don't prevent your workshops from being built (due to lack of blocks)."
-"quickfort run,orders library/dreamfort.csv -n /services2",# Run when the services levels have been dug out. Feel free to remove the orders for the ropes if you already brought them with you.
+"quickfort orders,run library/dreamfort.csv -n /services2",# Run when the services levels have been dug out. Feel free to remove the orders for the ropes if you already brought them with you.
-orders import basic,"# Run after the first migration wave, so you have dorfs to do all the basic tasks. Note that this is the ""orders"" plugin, not the ""quickfort orders"" command."
+orders import library/basic,"# Run after the first migration wave, so you have dorfs to do all the basic tasks. Note that this is the ""orders"" plugin, not the ""quickfort orders"" command."
-"quickfort run,orders library/dreamfort.csv -n /surface5","# Run when all marked trees on the surface are chopped down and walls and floors have been constructed, including the roof section over the future barracks."
+"quickfort orders,run library/dreamfort.csv -n /surface5","# Run when all marked trees on the surface are chopped down and walls and floors have been constructed, including the roof section over the future barracks."
 prioritize ConstructBuilding,# Run when you see the bridges ready to be built so the busy masons come and build them.
-"quickfort run,orders library/dreamfort.csv -n /surface6",# Run when at least the beehives and weapon rack are constructed and you have linked all levers to their respective bridges.
+"quickfort orders,run library/dreamfort.csv -n /surface6",# Run when at least the beehives and weapon rack are constructed and you have linked all levers to their respective bridges.
-"quickfort run,orders library/dreamfort.csv -n /surface7",# Run after the surface walls are completed and any marked trees are chopped down.
+"quickfort orders,run library/dreamfort.csv -n /surface7",# Run after the surface walls are completed and any marked trees are chopped down.
 ""
 -- Plumbing --
 "This is a good time to fill your well cisterns, either with a bucket brigade or by routing water from a freshwater stream or an aquifer (see the library/aquifer_tap.csv blueprint for help with this)."
 "Also consider bringing magma up to your services level so you can replace the forge and furnaces on your industry level with more powerful magma versions. This is especially important if your embark has insufficient trees to convert into charcoal. Keep in mind that moving magma is a tricky process and can take a long time. Don't forget to continue making progress through the checklist! If you choose to use magma, I suggest getting it in place before importing the military and smelting automation orders since they make heavy use of furnaces and forges."
 ""
 -- Mature fort (third migration wave onward) --
-orders import furnace,# Automated production of basic furnace-related items. Don't forget to create a sand collection zone (or remove the sand- and glass-related orders if you have no sand).
+orders import library/furnace,# Automated production of basic furnace-related items. Don't forget to create a sand collection zone (or remove the sand- and glass-related orders if you have no sand).
-"quickfort run,orders library/dreamfort.csv -n /suites2",# Run when the suites level has been dug out.
+"quickfort orders,run library/dreamfort.csv -n /suites2",# Run when the suites level has been dug out.
-"quickfort run,orders library/dreamfort.csv -n /surface8","# Run if/when you need longer trap corridors on the surface for larger sieges, anytime after you run /surface7."
+"quickfort orders,run library/dreamfort.csv -n /surface8","# Run if/when you need longer trap corridors on the surface for larger sieges, anytime after you run /surface7."
-"quickfort run,orders library/dreamfort.csv -n /apartments2",# Run when the first apartment level has been dug out.
+"quickfort orders,run library/dreamfort.csv -n /apartments2",# Run when the first apartment level has been dug out.
-"quickfort run,orders library/dreamfort.csv -n /services3","# Run after the dining table and chair, weapon rack, and archery targets have been constructed. Also wait until after you complete /surface7, though, because surface defenses are more important than a grand dining hall."
+"quickfort orders,run library/dreamfort.csv -n /services3","# Run after the dining table and chair, weapon rack, and archery targets have been constructed. Also wait until after you complete /surface7, though, because surface defenses are more important than a grand dining hall."
-"quickfort run,orders library/dreamfort.csv -n /guildhall2",# Run when the guildhall level has been dug out.
+"quickfort orders,run library/dreamfort.csv -n /guildhall2",# Run when the guildhall level has been dug out.
-"quickfort run,orders library/dreamfort.csv -n ""/guildhall3, /guildhall4""",# Optionally run after /guildhall2 to build default furnishings and declare a library and temple.
+"quickfort orders,run library/dreamfort.csv -n ""/guildhall3, /guildhall4""",# Optionally run after /guildhall2 to build default furnishings and declare a library and temple.
-"quickfort run,orders library/dreamfort.csv -n /apartments3",# Run when all beds have been constructed on the first apartments level.
+"quickfort orders,run library/dreamfort.csv -n /apartments3",# Run when all beds have been constructed on the first apartments level.
-"quickfort run,orders library/dreamfort.csv -n /farming4",# Run once you have a cache of potash.
+"quickfort orders,run library/dreamfort.csv -n /farming4",# Run once you have a cache of potash.
-orders import military,# Automated production of military equipment. Turn on automelt in the meltables piles on the industry level to automatically upgrade all metal military equipment to masterwork quality. These orders are optional if you are not using a military.
+orders import library/military,# Automated production of military equipment. Turn on automelt in the meltables piles on the industry level to automatically upgrade all metal military equipment to masterwork quality. These orders are optional if you are not using a military.
-orders import smelting,# Automated production of all types of metal bars.
+orders import library/smelting,# Automated production of all types of metal bars.
-"quickfort run,orders library/dreamfort.csv -n /services4","# Run when you need a jail, anytime after the restraints are placed from /services3."
+"quickfort orders,run library/dreamfort.csv -n /services4","# Run when you need a jail, anytime after the restraints are placed from /services3."
-orders import rockstock,# Maintains a small stock of all types of rock furniture.
+orders import library/rockstock,# Maintains a small stock of all types of rock furniture.
-orders import glassstock,# Maintains a small stock of all types of glass furniture and parts (only import if you have sand).
+orders import library/glassstock,# Maintains a small stock of all types of glass furniture and parts (only import if you have sand).
 ""
 -- Repeat for each remaining apartments level as needed --
-"quickfort run,orders library/dreamfort.csv -n /apartments2",# Run when the apartment level has been dug out.
+"quickfort orders,run library/dreamfort.csv -n /apartments2",# Run when the apartment level has been dug out.
-"quickfort run,orders library/dreamfort.csv -n /apartments3",# Run when all beds have been constructed.
+"quickfort orders,run library/dreamfort.csv -n /apartments3",# Run when all beds have been constructed.
 burial -pets,# Run once the coffins are placed to set them to allow for burial.
 
 See this checklist online at https://docs.google.com/spreadsheets/d/13PVZ2h3Mm3x_G1OXQvwKd7oIR2lK4A1Ahf6Om1kFigw/edit#gid=1459509569
@@ -232,6 +229,7 @@ https://drive.google.com/file/d/1Et42JTzeYK23iI5wrPMsFJ7lUXwVBQob/view?usp=shari
 [PET:2:BIRD_GOOSE:FEMALE:STANDARD]
 [PET:2:BIRD_GOOSE:MALE:STANDARD]
 #meta label(all_orders) hidden() references all blueprints that generate orders; for testing only
+/surface1
 /surface2
 /surface3
 /surface4
@@ -239,27 +237,25 @@ https://drive.google.com/file/d/1Et42JTzeYK23iI5wrPMsFJ7lUXwVBQob/view?usp=shari
 /surface6
 /surface7
 /surface8
+/farming1
 /farming2
 /farming3
 /farming4
+/industry1
 /industry2
+/services1
 /services2
 /services3
 /services4
+/guildhall1
 /guildhall2
+/guildhall3
+/guildhall4
+/suites1
 /suites2
-/apartments2
+/apartments1 repeat(>5)
-/apartments2
+/apartments2 repeat(>5)
-/apartments2
+/apartments3 repeat(>5)
-/apartments2
-/apartments2
-/apartments2
-/apartments3
-/apartments3
-/apartments3
-/apartments3
-/apartments3
-/apartments3
 #notes label(surface_help)
 Sets up a protected entrance to your fort in a flat area on the surface.
 Screenshot: https://drive.google.com/file/d/1YL_vQJLB2YnUEFrAg9y3HEdFq3Wpw9WP
@@ -1831,7 +1827,6 @@ Workshops:
 Manual steps you have to take:
 - Assign minecarts to your quantum stockpile hauling routes
 "- Give from the ""Goods"" quantum stockpile to the jugs, pots, and bags stockpiles on the farming level"
-- Copy the fort automation manager orders (the .json files) from hack/examples/orders/ and put them in your dfhack-config/orders/ directory.
 ""
 Optional manual steps you can take:
 - Restrict the Mechanic's workshop to only allow skilled workers so unskilled trap-resetters won't be tasked to build mechanisms.
@@ -1847,11 +1842,11 @@ Industry Walkthrough:
 ""
 "3) Once the area is dug out, run /industry2. Remember to assign minecarts to to your quantum stockpile hauling routes, and if the farming level is already built, give from the ""Goods"" quantum stockpile (the one on the left) to the jugs, pots, and bags stockpiles on the farming level."
 ""
-"4) Once you have enough dwarves to do maintenance tasks (that is, after the first or second migration wave), run ""orders import basic"" to use the provided basic.json to take care of your fort's basic needs, such as food, booze, and raw material processing."
+"4) Once you have enough dwarves to do maintenance tasks (that is, after the first or second migration wave), run ""orders import library/basic"" to use the provided basic.json to take care of your fort's basic needs, such as food, booze, and raw material processing."
 ""
 "5) If you want to automatically melt goblinite and other low-quality weapons and armor, mark the south-east stockpiles for auto-melt. If you don't have a high density of trees to make into charcoal, though, be sure to route magma to the level beneath this one and replace the forge and furnaces with magma equivalents."
 ""
-"6) Once you have magma furnaces (or abundant fuel) and more dwarves, run ""orders import furnace"", ""orders import military"", and ""orders import smelting"" to import the remaining fort automation orders. The military orders are optional if you are not planning to have a military, of course."
+"6) Once you have magma furnaces (or abundant fuel) and more dwarves, run ""orders import library/furnace"", ""orders import library/military"", and ""orders import library/smelting"" to import the remaining fort automation orders. The military orders are optional if you are not planning to have a military, of course."
 ""
 "7) At any time, feel free to build extra workshops or designate custom stockpiles in the unused space in the top and bottom right. The space is there for you to use!"
 "#dig label(industry1) start(18; 18; central stairs) message(Once the area is dug out, continue with /industry2.)"
@@ -1970,7 +1965,7 @@ query/industry_query
 - assign minecarts to to your quantum stockpile hauling routes (use ""assign-minecarts all"")
 - if the farming level is already built, give from the ""Goods"" quantum stockpile to the jugs, pots, and bags stockpiles on the farming level
 - if you want to automatically melt goblinite and other low-quality weapons and armor, mark the south-east stockpiles for auto-melt
-- once you have enough dwarves, run ""orders import basic"" to automate your fort's basic needs (see /industry_help for more info on this file)
+- once you have enough dwarves, run ""orders import library/basic"" to automate your fort's basic needs (see /industry_help for more info on this file)
 - optionally, restrict the labors for your Craftsdwarf's and Mechanic's workshops as per the guidance in /industry_help)"
 
 
@@ -2382,7 +2377,7 @@ query_jail/services_query_jail
 ,`,`,`,`,`,,`,`,`,`,`
 ,`,`,`,`,`,,`,`,`,`,`
 
-"#query label(services_query_dining) start(18; 18) message(The tavern is restricted to residents only by default. If you'd like your tavern to attract vistors, please go to the (l)ocation menu and change the restriction.) set up dining room/tavern and barracks"
+"#query label(services_query_dining) start(18; 18) hidden() message(The tavern is restricted to residents only by default. If you'd like your tavern to attract vistors, please go to the (l)ocation menu and change the restriction.) set up dining room/tavern and barracks"
 
 ,`,`,`,,`,`,`,,`,`,`,,`,`,`,,`,,`,`,`
 ,`,`,`,,`,`,`,,`,`,`,,`,`,`,`,`,`,`,`,`
@@ -2418,7 +2413,7 @@ query_jail/services_query_jail
 ,`,`,`,`,`,,`,`,`,`,`
 ,`,`,`,`,`,,`,`,`,`,`
 
-#query label(services_query_rented_rooms) start(18; 18) attach rented rooms to tavern
+#query label(services_query_rented_rooms) start(18; 18) hidden() attach rented rooms to tavern
 
 ,r&l-&,r&l-&,r&l-&,,r&l-&,r&l-&,r&l-&,,r&l-&,r&l-&,r&l-&,,`,`,`,,`,,`,`,`
 ,`,`,`,,`,`,`,,`,`,`,,`,`,`,`,`,`,`,`,`

data/examples/init/onMapLoad_dreamfort.init

@@ -37,17 +37,14 @@ enable automelt
 
 # creates manager orders to produce replacements for worn clothing
 enable tailor
-tailor enable
 
 # auto-assigns nesting birds to nestbox zones and protects fertile eggs from
 # being cooked/eaten
-enable zone nestboxes
+enable zone autonestbox nestboxes
-autonestbox start
 
 # manages seed stocks
 enable seedwatch
 seedwatch all 30
-seedwatch start
 
 # ensures important tasks get assigned to workers.
 # otherwise these job types can get ignored in busy forts.
@@ -65,6 +62,7 @@ prioritize -a --reaction-name=TAN_A_HIDE CustomReaction
 # feel free to change this to "target 0 0 0 0" if you don't expect to want to raise
 # any animals not listed here -- you can always change it anytime during the game
 # later if you change your mind.
+on-new-fortress enable autobutcher
 on-new-fortress autobutcher target 2 2 2 2 new
 # dogs and cats. You should raise the limits for dogs if you will be training them
 # for hunting or war.
@@ -82,5 +80,5 @@ on-new-fortress autobutcher target 2 2 4 2 ALPACA SHEEP LLAMA
 on-new-fortress autobutcher target 5 5 6 2 PIG
 # butcher all unprofitable animals
 on-new-fortress autobutcher target 0 0 0 0 HORSE YAK DONKEY WATER_BUFFALO GOAT CAVY BIRD_DUCK BIRD_GUINEAFOWL
-# start it up!
+# watch for new animals
-on-new-fortress autobutcher start; autobutcher watch all; autobutcher autowatch
+on-new-fortress autobutcher autowatch

data/init/dfhack.default.init

@@ -0,0 +1,8 @@
+# Default DFHack commands to run on program init
+
+# Please do not edit this file directly. It will be overwritten with new
+# defaults when you update DFHack. Instead, add your configuration to
+# dfhack-config/init/dfhack.init
+
+script hack/init/dfhack.keybindings.init
+script hack/init/dfhack.tools.init

data/init/dfhack.keybindings.init

@@ -1,7 +1,17 @@
+# Default DFHack keybindings
+
+# Please do not edit this file directly. It will be overwritten with new
+# defaults when you update DFHack. Instead, add your configuration to
+# dfhack-config/init/dfhack.init
+
 ##############################
 # Generic dwarfmode bindings #
 ##############################
 
+# the GUI command launcher (two bindings since some keyboards don't have `)
+keybinding add ` gui/launcher
+keybinding add Ctrl-Shift-D gui/launcher
+
 # show all current key bindings
 keybinding add Ctrl-F1 hotkeys
 keybinding add Alt-F1 hotkeys
@@ -34,7 +44,7 @@ keybinding add Ctrl-K autodump-destroy-item
 # quicksave, only in main dwarfmode screen and menu page
 keybinding add Ctrl-Alt-S@dwarfmode/Default quicksave
 
-# gui/quickfort script - apply pre-made blueprints to the map
+# apply blueprints to the map (Alt-F for compatibility with LNP Quickfort)
 keybinding add Ctrl-Shift-Q@dwarfmode gui/quickfort
 keybinding add Alt-F@dwarfmode gui/quickfort
 
@@ -68,6 +78,12 @@ keybinding add Ctrl-Shift-R view-unit-reports
 # view extra unit information
 keybinding add Alt-I@dwarfmode/ViewUnits|unitlist gui/unit-info-viewer
 
+# set workorder item details (on workorder details screen press D again)
+keybinding add D@workquota_details gui/workorder-details
+
+# boost priority of jobs related to the selected entity
+keybinding add Alt-N do-job-now
+
 ##############################
 # Generic adv mode bindings  #
 ##############################
@@ -161,137 +177,3 @@ keybinding add Shift-B@pet/List/Unit "gui/autobutcher"
 
 # view pathable tiles from active cursor
 keybinding add Alt-Shift-P@dwarfmode/LookAround gui/pathable
-
-############################
-# UI and game logic tweaks #
-############################
-
-# stabilize the cursor of dwarfmode when switching menus
-tweak stable-cursor
-
-# stop stacked liquid/bar/thread/cloth items from lasting forever
-# if used in reactions that use only a fraction of the dimension.
-# might be fixed by DF
-# tweak fix-dimensions
-
-# make reactions requiring containers usable in advmode - the issue is
-# that the screen asks for those reagents to be selected directly
-tweak advmode-contained
-
-# support Shift-Enter in Trade and Move Goods to Depot screens for faster
-# selection; it selects the current item or stack and scrolls down one line
-tweak fast-trade
-
-# stop the right list in military->positions from resetting to top all the time
-tweak military-stable-assign
-# in same list, color units already assigned to squads in brown & green
-tweak military-color-assigned
-
-# make crafted cloth items wear out with time like in old versions (bug 6003)
-tweak craft-age-wear
-
-# stop adamantine clothing from wearing out (bug 6481)
-#tweak adamantine-cloth-wear
-
-# Add "Select all" and "Deselect all" options to farm plot menus
-tweak farm-plot-select
-
-# Add Shift-Left/Right controls to import agreement screen
-tweak import-priority-category
-
-# Fixes a crash in the work order contition material list (bug 9905).
-tweak condition-material
-
-# Adds an option to clear currently-bound hotkeys
-tweak hotkey-clear
-
-# Allows lowercase letters in embark profile names, and allows exiting the name prompt without saving
-tweak embark-profile-name
-
-# Reduce performance impact of temperature changes
-tweak fast-heat 100
-
-# Misc. UI tweaks
-tweak block-labors              # Prevents labors that can't be used from being toggled
-tweak burrow-name-cancel
-tweak cage-butcher
-tweak civ-view-agreement
-tweak do-job-now
-tweak eggs-fertile
-tweak fps-min
-tweak hide-priority
-tweak kitchen-prefs-all
-tweak kitchen-prefs-empty
-tweak max-wheelbarrow
-tweak partial-items
-tweak shift-8-scroll
-tweak stone-status-all
-tweak title-start-rename
-tweak tradereq-pet-gender
-
-###########################
-# Globally acting plugins #
-###########################
-
-# Display DFHack version on title screen
-enable title-version
-
-# Dwarf Manipulator (simple in-game Dwarf Therapist replacement)
-enable manipulator
-
-# Search tool in various screens (by falconne)
-enable search
-
-# Improved build material selection interface (by falconne)
-enable automaterial
-
-# Other interface improvement tools
-enable \
- confirm \
- dwarfmonitor \
- mousequery \
- autogems \
- autodump \
- automelt \
- autotrade \
- buildingplan \
- resume \
- trackstop \
- zone \
- stocks \
- autochop \
- stockpiles
-#end a line with a backslash to make it continue to the next line. The \ is deleted for the final command.
-# Multiline commands are ONLY supported for scripts like dfhack.init. You cannot do multiline command manually on the DFHack console.
-# You cannot extend a commented line.
-# You can comment out the extension of a line.
-
-# enable mouse controls and sand indicator in embark screen
-embark-tools enable sticky sand mouse
-
-# enable option to enter embark assistant
-enable embark-assistant
-
-###########
-# Scripts #
-###########
-
-# write extra information to the gamelog
-modtools/extra-gamelog enable
-
-# extended status screen (bedrooms page)
-enable gui/extended-status
-
-# add information to item viewscreens
-view-item-info enable
-
-# a replacement for the "load game" screen
-gui/load-screen enable
-
-##############################
-# Extra DFHack command files #
-##############################
-
-# Create a file named "onLoad.init" to run commands when a world is loaded
-# and/or create a file named "onMapLoad.init" to run commands when a map is
-# loaded. See the hack/examples/init/ directory for useful pre-made init files.

data/init/dfhack.tools.init

@@ -0,0 +1,131 @@
+# Default DFHack tool configuration
+
+# Please do not edit this file directly. It will be overwritten with new
+# defaults when you update DFHack. Instead, add your configuration to
+# dfhack-config/init/dfhack.init
+
+############################
+# UI and game logic tweaks #
+############################
+
+# stabilize the cursor of dwarfmode when switching menus
+tweak stable-cursor
+
+# stop stacked liquid/bar/thread/cloth items from lasting forever
+# if used in reactions that use only a fraction of the dimension.
+# might be fixed by DF
+# tweak fix-dimensions
+
+# make reactions requiring containers usable in advmode - the issue is
+# that the screen asks for those reagents to be selected directly
+tweak advmode-contained
+
+# support Shift-Enter in Trade and Move Goods to Depot screens for faster
+# selection; it selects the current item or stack and scrolls down one line
+tweak fast-trade
+
+# stop the right list in military->positions from resetting to top all the time
+tweak military-stable-assign
+# in same list, color units already assigned to squads in brown & green
+tweak military-color-assigned
+
+# make crafted cloth items wear out with time like in old versions (bug 6003)
+tweak craft-age-wear
+
+# stop adamantine clothing from wearing out (bug 6481)
+#tweak adamantine-cloth-wear
+
+# Add "Select all" and "Deselect all" options to farm plot menus
+tweak farm-plot-select
+
+# Add Shift-Left/Right controls to import agreement screen
+tweak import-priority-category
+
+# Fixes a crash in the work order contition material list (bug 9905).
+tweak condition-material
+
+# Adds an option to clear currently-bound hotkeys
+tweak hotkey-clear
+
+# Allows lowercase letters in embark profile names, and allows exiting the name prompt without saving
+tweak embark-profile-name
+
+# Reduce performance impact of temperature changes
+tweak fast-heat 100
+
+# Misc. UI tweaks
+tweak block-labors              # Prevents labors that can't be used from being toggled
+tweak burrow-name-cancel
+tweak cage-butcher
+tweak civ-view-agreement
+tweak do-job-now
+tweak eggs-fertile
+tweak fps-min
+tweak hide-priority
+tweak kitchen-prefs-all
+tweak kitchen-prefs-empty
+tweak max-wheelbarrow
+tweak partial-items
+tweak shift-8-scroll
+tweak stone-status-all
+tweak title-start-rename
+tweak tradereq-pet-gender
+
+###########################
+# Globally acting plugins #
+###########################
+
+# Display DFHack version on title screen
+enable title-version
+
+# Dwarf Manipulator (simple in-game Dwarf Therapist replacement)
+enable manipulator
+
+# Search tool in various screens (by falconne)
+enable search
+
+# Improved build material selection interface (by falconne)
+enable automaterial
+
+# Other interface improvement tools
+enable \
+ confirm \
+ dwarfmonitor \
+ mousequery \
+ autogems \
+ autodump \
+ automelt \
+ autotrade \
+ buildingplan \
+ resume \
+ trackstop \
+ zone \
+ stocks \
+ autochop \
+ stockpiles
+#end a line with a backslash to make it continue to the next line. The \ is deleted for the final command.
+# Multiline commands are ONLY supported for scripts like dfhack.init. You cannot do multiline command manually on the DFHack console.
+# You cannot extend a commented line.
+# You can comment out the extension of a line.
+
+# enable mouse controls and sand indicator in embark screen
+embark-tools enable sticky sand mouse
+
+# enable option to enter embark assistant
+enable embark-assistant
+
+###########
+# Scripts #
+###########
+
+# write extra information to the gamelog
+modtools/extra-gamelog enable
+
+# extended status screen (bedrooms page)
+enable gui/extended-status
+
+# add information to item viewscreens
+view-item-info enable
+
+# a replacement for the "load game" screen
+gui/load-screen enable

data/init/onLoad.default.init

@@ -0,0 +1,5 @@
+# Default DFHack commands to run when a world is loaded
+
+# Please do not edit this file directly. It will be overwritten with new
+# defaults when you update DFHack. Instead, add your configuration to
+# dfhack-config/init/onLoad.init

data/init/onMapLoad.default.init

@@ -0,0 +1,6 @@
+# Default DFHack commands to run when a map is loaded, either in
+# adventure or fort mode.
+
+# Please do not edit this file directly. It will be overwritten with new
+# defaults when you update DFHack. Instead, add your configuration to
+# dfhack-config/init/onMapLoad.init

data/init/onMapUnload.default.init

@@ -0,0 +1,5 @@
+# Default DFHack commands to run when a map is unloaded
+
+# Please do not edit this file directly. It will be overwritten with new
+# defaults when you update DFHack. Instead, add your configuration to
+# dfhack-config/init/onMapUnload.init

data/init/onUnload.default.init

@@ -0,0 +1,5 @@
+# Default DFHack commands to run when a world is unloaded
+
+# Please do not edit this file directly. It will be overwritten with new
+# defaults when you update DFHack. Instead, add your configuration to
+# dfhack-config/init/onUnload.init

data/professions/Chef

@@ -1,4 +1,4 @@
-NAME Chef
+NAME library/Chef
 BUTCHER
 TANNER
 COOK

data/professions/Craftsdwarf

@@ -1,4 +1,4 @@
-NAME Craftsdwarf
+NAME library/Craftsdwarf
 WOOD_CRAFT
 STONE_CRAFT
 BONE_CARVE

data/professions/Doctor

@@ -1,4 +1,4 @@
-NAME Doctor
+NAME library/Doctor
 ANIMALCARE
 DIAGNOSE
 SURGERY

data/professions/Farmer

@@ -1,4 +1,4 @@
-NAME Farmer
+NAME library/Farmer
 PLANT
 MILLER
 BREWER

data/professions/Fisherdwarf

@@ -1,4 +1,4 @@
-NAME Fisherdwarf
+NAME library/Fisherdwarf
 FISH
 CLEAN_FISH
 DISSECT_FISH

data/professions/Hauler

@@ -1,4 +1,4 @@
-NAME Hauler
+NAME library/Hauler
 FEED_WATER_CIVILIANS
 SIEGEOPERATE
 MECHANIC

data/professions/Laborer

@@ -1,4 +1,4 @@
-NAME Laborer
+NAME library/Laborer
 SOAP_MAKER
 BURN_WOOD
 POTASH_MAKING

data/professions/Marksdwarf

@@ -1,4 +1,4 @@
-NAME Marksdwarf
+NAME library/Marksdwarf
 MECHANIC
 HAUL_STONE
 HAUL_WOOD

data/professions/Mason

@@ -1,4 +1,4 @@
-NAME Mason
+NAME library/Mason
 MASON
 CUT_GEM
 ENCRUST_GEM

data/professions/Meleedwarf

@@ -1,4 +1,4 @@
-NAME Meleedwarf
+NAME library/Meleedwarf
 RECOVER_WOUNDED
 MECHANIC
 HAUL_STONE

data/professions/Migrant

@@ -1,4 +1,4 @@
-NAME Migrant
+NAME library/Migrant
 FEED_WATER_CIVILIANS
 SIEGEOPERATE
 MECHANIC

data/professions/Miner

@@ -1,4 +1,4 @@
-NAME Miner
+NAME library/Miner
 MINE
 DETAIL
 RECOVER_WOUNDED

data/professions/Outdoorsdwarf

@@ -1,4 +1,4 @@
-NAME Outdoorsdwarf
+NAME library/Outdoorsdwarf
 CARPENTER
 BOWYER
 CUTWOOD

data/professions/Smith

@@ -1,4 +1,4 @@
-NAME Smith
+NAME library/Smith
 FORGE_WEAPON
 FORGE_ARMOR
 FORGE_FURNITURE

data/professions/StartManager

@@ -1,4 +1,4 @@
-NAME StartManager
+NAME library/StartManager
 CUTWOOD
 ANIMALCARE
 DIAGNOSE

data/professions/Tailor

@@ -1,4 +1,4 @@
-NAME Tailor
+NAME library/Tailor
 DYER
 LEATHER
 WEAVER

dfhack-config/init/default.dfhack.init

@@ -0,0 +1,7 @@
+# Load DFHack defaults.
+#
+# If you delete this file, it will reappear when you restart DFHack.
+# Instead, please comment out the following line if you do not want DFHack to
+# load its default configuration.
+
+script hack/init/dfhack.default.init

dfhack-config/init/default.onLoad.init

@@ -0,0 +1,7 @@
+# Load DFHack defaults.
+#
+# If you delete this file, it will reappear when you restart DFHack.
+# Instead, please comment out the following line if you do not want DFHack to
+# load its default configuration.
+
+script hack/init/onLoad.default.init

dfhack-config/init/default.onMapLoad.init

@@ -0,0 +1,7 @@
+# Load DFHack defaults.
+#
+# If you delete this file, it will reappear when you restart DFHack.
+# Instead, please comment out the following line if you do not want DFHack to
+# load its default configuration.
+
+script hack/init/onMapLoad.default.init

dfhack-config/init/default.onMapUnload.init

@@ -0,0 +1,7 @@
+# Load DFHack defaults.
+#
+# If you delete this file, it will reappear when you restart DFHack.
+# Instead, please comment out the following line if you do not want DFHack to
+# load its default configuration.
+
+script hack/init/onMapUnload.default.init

dfhack-config/init/default.onUnload.init

@@ -0,0 +1,7 @@
+# Load DFHack defaults.
+#
+# If you delete this file, it will reappear when you restart DFHack.
+# Instead, please comment out the following line if you do not want DFHack to
+# load its default configuration.
+
+script hack/init/onUnload.default.init

dfhack-config/init/dfhack.init

@@ -0,0 +1,5 @@
+# This file runs when DFHack is initialized, when Dwarf Fortress is first
+# started, before any world or save data is loaded.
+#
+# You can extend or override DFHack's default configuration by adding commands
+# to this file.

dfhack-config/init/onLoad.init

@@ -0,0 +1,6 @@
+# This file runs when a world is loaded. This happens when you open a save file
+# in fort, adventure, or legends mode. If a fort is being loaded, this file runs
+# before any onMapLoad.init files.
+#
+# You can extend or override DFHack's default configuration by adding commands
+# to this file.

dfhack-config/init/onMapLoad.init

@@ -0,0 +1,5 @@
+# This file runs when a map is loaded in adventure or fort mode, after any
+# onLoad.init files (which run earlier, when the world is loaded).
+#
+# You can extend or override DFHack's default configuration by adding commands
+# to this file.

dfhack-config/init/onMapUnload.init

@@ -0,0 +1,5 @@
+# This file runs when a fortress map is unloaded, before any onUnload.init files
+# (which run later, when the world is unloaded).
+#
+# You can extend or override DFHack's default configuration by adding commands
+# to this file.

dfhack-config/init/onUnload.init

@@ -0,0 +1,4 @@
+# This file runs when a world is unloaded.
+#
+# You can extend or override DFHack's default configuration by adding commands
+# to this file.

docs/Core.rst

@@ -224,7 +224,8 @@ To set keybindings, use the built-in ``keybinding`` command. Like any other
 command it can be used at any time from the console, but bindings are not
 remembered between runs of the game unless re-created in `dfhack.init`.
 
-Currently, any combinations of Ctrl/Alt/Shift with A-Z, 0-9, or F1-F12 are supported.
+Currently, any combinations of Ctrl/Alt/Shift with A-Z, 0-9, F1-F12 or \`
+are supported.
 
 Possible ways to call the command:
 
@@ -366,16 +367,27 @@ The following commands are *not* built-in, but offer similarly useful functions.
 * `repeat`
 
 
+.. _dfhack-config:
+
+Configuration Files
+===================
+
+Most DFHack settings can be changed by modifying files in the ``dfhack-config``
+folder (which is in the DF folder). The default versions of these files, if they
+exist, are in ``dfhack-config/default`` and are installed when DFHack starts if
+necessary.
+
 .. _init-files:
 
 Init Files
-==========
+----------
 
 .. contents::
    :local:
 
 DFHack allows users to automatically run commonly-used DFHack commands
-when DF is first loaded, when a game is loaded, and when a game is unloaded.
+when DF is first loaded, when a world is loaded, when a map is loaded, when a
+map is unloaded, and when a world is unloaded.
 
 Init scripts function the same way they would if the user manually typed
 in their contents, but are much more convenient.  In order to facilitate
@@ -385,32 +397,33 @@ save-specific init files in the save folders.
 
 DFHack looks for init files in three places each time they could be run:
 
-#. The main DF directory
+#. The :file:`dfhack-config/init` subdirectory in the main DF directory
 #. :file:`data/save/{world}/raw`, where ``world`` is the current save, and
 #. :file:`data/save/{world}/raw/objects`
 
-When reading commands from dfhack.init or with the `script` command, if the final
+For each of those directories, all matching init files will be executed in
-character on a line is a backslash then the next uncommented line is considered a
+alphabetical order.
-continuation of that line, with the backslash deleted.  Commented lines are skipped,
-so it is possible to comment out parts of a command with the ``#`` character.
 
+Before running matched init scripts in any of those locations, the
+:file:`dfhack-config/init/default.*` file that matches the event will be run to
+load DFHack defaults. Only the :file:`dfhack-config/init` directory is checked
+for this file, not any :file:`raw` directories. If you want DFHack to load
+without running any of its default configuration commands, edit the
+:file:`dfhack-config/init/default.*` files and comment out the commands you see
+there.
 
-.. _dfhack.init:
+When reading commands from the init files or with the `script` command, if the
+final character on a line is a backslash then the next uncommented line is
+considered a continuation of that line, with the backslash deleted.  Commented
+lines are skipped, so it is possible to comment out parts of a command with the
+``#`` character.
 
-dfhack*.init
+.. _dfhack.init:
-------------
-If your DF folder contains at least one file named ``dfhack*.init``
-(where ``*`` is a placeholder for any string), then all such files
-are executed in alphabetical order when DF is first started.
-
-DFHack is distributed with :download:`/dfhack.init-example` as an example
-with an up-to-date collection of basic commands; mostly setting standard
-keybindings and `enabling <enable>` plugins.  You are encouraged to look
-through this file to learn which features it makes available under which
-key combinations.  You may also customise it and rename it to ``dfhack.init``.
 
-If your DF folder does not contain any ``dfhack*.init`` files, the example
+dfhack\*.init
-will be run as a fallback.
+.............
+On startup, DFHack looks for files of the form ``dfhack*.init`` (where ``*`` is
+a placeholder for any string, including the empty string).
 
 These files are best used for keybindings and enabling persistent plugins
 which do not require a world to be loaded.
@@ -418,51 +431,49 @@ which do not require a world to be loaded.
 
 .. _onLoad.init:
 
-onLoad*.init
+onLoad\*.init
-------------
+.............
 When a world is loaded, DFHack looks for files of the form ``onLoad*.init``,
 where ``*`` can be any string, including the empty string.
 
-All matching init files will be executed in alphabetical order.
 A world being loaded can mean a fortress, an adventurer, or legends mode.
 
 These files are best used for non-persistent commands, such as setting
 a `fix <scripts-fix>` script to run on `repeat`.
 
 
-.. _onUnload.init:
+.. _onMapLoad.init:
 
-onUnload*.init
+onMapLoad\*.init
---------------
+................
-When a world is unloaded, DFHack looks for files of the form ``onUnload*.init``.
+When a map is loaded, either in adventure or fort mode, DFHack looks for files
-Again, these files may be in any of the above three places.
+of the form ``onMapLoad*.init``, where ``*`` can be any string, including the
-All matching init files will be executed in alphebetical order.
+empty string.
 
-Modders often use such scripts to disable tools which should not affect
+These files are best used for commands that are only relevant once there is a
-an unmodded save.
+game map loaded.
 
-.. _other_init_files:
 
-Other init files
+.. _onMapUnload.init:
-----------------
+.. _onUnload.init:
 
-* ``onMapLoad*.init`` and ``onMapUnload*.init`` are run when a map,
+onMapUnload\*.init and onUnload\*.init
-  distinct from a world, is loaded.  This is good for map-affecting
+......................................
-  commands (e.g. `clean`), or avoiding issues in Legends mode.
+When a map or world is unloaded, DFHack looks for files of the form
+``onMapUnload*.init`` or ``onUnload*.init``, respectively.
 
-* Any lua script named ``raw/init.d/*.lua``, in the save or main DF
+Modders often use unload init scripts to disable tools which should not run
-  directory, will be run when any world or that save is loaded.
+after a modded save is unloaded.
 
 
-.. _dfhack-config:
+.. _other_init_files:
 
-Configuration Files
+raw/init.d/\*.lua
-===================
+.................
+
+Any lua script named ``raw/init.d/*.lua``, in the save or main DF directory,
+will be run when any world or that save is loaded.
 
-Some DFHack settings can be changed by modifying files in the ``dfhack-config``
-folder (which is in the DF folder). The default versions of these files, if they
-exist, are in ``dfhack-config/default`` and are installed when DFHack starts if
-necessary.
 
 .. _script-paths:
 

docs/Dev-intro.rst

@@ -22,7 +22,7 @@ Plugins
 
 DFHack plugins are written in C++ and located in the ``plugins`` folder.
 Currently, documentation on how to write plugins is somewhat sparse. There are
-templates that you can use to get started in the ``plugins/skeleton``
+templates that you can use to get started in the ``plugins/examples``
 folder, and the source code of existing plugins can also be helpful.
 
 If you want to compile a plugin that you have just added, you will need to add a
@@ -35,7 +35,7 @@ other commands).
 
 Plugins can also register handlers to run on every tick, and can interface with
 the built-in `enable` and `disable` commands. For the full plugin API, see the
-skeleton plugins or ``PluginManager.cpp``.
+example ``skeleton`` plugin or ``PluginManager.cpp``.
 
 Installed plugins live in the ``hack/plugins`` folder of a DFHack installation,
 and the `load` family of commands can be used to load a recompiled plugin

docs/Lua API.rst

@@ -525,6 +525,20 @@ Input & Output
   lock. Using an explicit ``dfhack.with_suspend`` will prevent
   this, forcing the function to block on input with lock held.
 
+* ``dfhack.getCommandHistory(history_id, history_filename)``
+
+  Returns the list of strings in the specified history. Intended to be used by
+  GUI scripts that don't have access to a console and so can't use
+  ``dfhack.lineedit``. The ``history_id`` parameter is some unique string that
+  the script uses to identify its command history, such as the script's name. If
+  this is the first time the history with the given ``history_id`` is being
+  accessed, it is initialized from the given file.
+
+* ``dfhack.addCommandToHistory(history_id, history_filename, command)``
+
+  Adds a command to the specified history and saves the updated history to the
+  specified file.
+
 * ``dfhack.interpreter([prompt[,history_filename[,env]]])``
 
   Starts an interactive lua interpreter, using the specified prompt
@@ -838,6 +852,7 @@ can be omitted.
 * ``dfhack.getGitXmlExpectedCommit()``
 * ``dfhack.gitXmlMatch()``
 * ``dfhack.isRelease()``
+* ``dfhack.isPrerelease()``
 
   Return information about the DFHack build in use.
 
@@ -891,7 +906,6 @@ can be omitted.
   from ``dfhack.TranslateName()``), use ``print(dfhack.df2console(text))`` to
   ensure proper display on all platforms.
 
-
 * ``dfhack.utf2df(string)``
 
   Convert a string from UTF-8 to DF's CP437 encoding.
@@ -3030,6 +3044,18 @@ parameters.
   function also verifies that the coordinates are valid for the current map and
   throws if they are not (unless ``skip_validation`` is set to true).
 
+* ``argparse.positiveInt(arg, arg_name)``
+
+  Throws if ``tonumber(arg)`` is not a positive integer; otherwise returns
+  ``tonumber(arg)``. If ``arg_name`` is specified, it is used to make error
+  messages more useful.
+
+* ``argparse.nonnegativeInt(arg, arg_name)``
+
+  Throws if ``tonumber(arg)`` is not a non-negative integer; otherwise returns
+  ``tonumber(arg)``. If ``arg_name`` is specified, it is used to make error
+  messages more useful.
+
 dumper
 ======
 
@@ -3043,6 +3069,87 @@ function:
   argument specifies the indentation step size in spaces. For
   the other arguments see the original documentation link above.
 
+helpdb
+======
+
+Unified interface for DFHack tool help text. Help text is read from the rendered
+text in ``hack/docs/docs/``. If no rendered text exists, help is read from the
+script sources (for scripts) or the string passed to the ``PluginCommand``
+initializer (for plugins). See `documentation` for details on how DFHack's help
+system works.
+
+The database is lazy-loaded when an API method is called. It rechecks its help
+sources for updates if an API method has not been called in the last 60 seconds.
+
+Each entry has several properties associated with it:
+
+- The entry name, which is the name of a plugin, script, or command provided by
+  a plugin.
+- The entry types, which can be ``builtin``, ``plugin``, and/or ``command``.
+  Entries for built-in commands (like ``ls`` or ``quicksave``) are both type
+  ``builtin`` and ``command``. Entries named after plugins are type ``plugin``,
+  and if that plugin also provides a command with the same name as the plugin,
+  then the entry is also type ``command``. Entry types are returned as a map
+  of one or more of the type strings to ``true``.
+- Short help, a the ~54 character description string.
+- Long help, the entire contents of the associated help file.
+- A list of tags that define the groups that the entry belongs to.
+
+* ``helpdb.is_entry(str)``, ``helpdb.is_entry(list)``
+
+  Returns whether the given string (or list of strings) is an entry (are all
+  entries) in the db.
+
+* ``helpdb.get_entry_types(entry)``
+
+  Returns the set (that is, a map of string to ``true``) of entry types for the
+  given entry.
+
+* ``helpdb.get_entry_short_help(entry)``
+
+  Returns the short (~54 character) description for the given entry.
+
+* ``helpdb.get_entry_long_help(entry)``
+
+  Returns the full help text for the given entry.
+
+* ``helpdb.get_entry_tags(entry)``
+
+  Returns the set of tag names for the given entry.
+
+* ``helpdb.is_tag(str)``, ``helpdb.is_tag(list)``
+
+  Returns whether the given string (or list of strings) is a (are all) valid tag
+  name(s).
+
+* ``helpdb.get_tags()``
+
+  Returns the full alphabetized list of valid tag names.
+
+* ``helpdb.get_tag_data(tag)``
+
+  Returns a list of entries that have the given tag. The returned table also
+  has a ``description`` key that contains the string description of the tag.
+
+* ``helpdb.search_entries([include[, exclude]])``
+
+  Returns a list of names for entries that match the given filters. The list is
+  alphabetized by their last path component, with populated path components
+  coming before null path components (e.g. ``autobutcher`` will immediately
+  follow ``gui/autobutcher``).
+  The optional ``include`` and ``exclude`` filter params are maps with the
+  following elements:
+
+  :str:   if a string, filters by the given substring. if a table of strings,
+          includes entry names that match any of the given substrings.
+  :tag:   if a string, filters by the given tag name. if a table of strings,
+          includes entries that match any of the given tags.
+  :entry_type: if a string, matches entries of the given type. if a table of
+          strings, includes entries that match any of the given types.
+
+  If ``include`` is ``nil`` or empty, then all entries are included. If
+  ``exclude`` is ``nil`` or empty, then no entries are filtered out.
+
 profiler
 ========
 
@@ -3242,24 +3349,27 @@ A module for reading custom tokens added to the raws by mods.
   The same as ``getToken(plantGrowthItem, token)`` but with a specified plant and growth, using growth name
   (e.g. "LEAVES") instead of number.
 
+It is recommended to prefix custom raw tokens with the name of your mod to avoid duplicate behaviour where
+two mods make callbacks that work on the same tag.
+
 Examples:
 
 * Using an eventful onReactionComplete hook, something for disturbing dwarven science::
 
-    if customRawTokens.getToken(reaction, "DFHACK_CAUSES_INSANITY") then
+    if customRawTokens.getToken(reaction, "EXAMPLE_MOD_CAUSES_INSANITY") then
         -- make unit who performed reaction go insane
 
 * Using an eventful onProjItemCheckMovement hook, a fast or slow-firing crossbow::
 
     -- check projectile distance flown is zero, get firer, etc...
-    local multiplier = tonumber(customRawTokens.getToken(bow, "DFHACK_FIRE_RATE_MULTIPLIER")) or 1
+    local multiplier = tonumber(customRawTokens.getToken(bow, "EXAMPLE_MOD_FIRE_RATE_MULTIPLIER")) or 1
     firer.counters.think_counter = firer.counters.think_counter * multiplier
 
 * Something for a script that prints help text about different types of units::
 
     local unit = dfhack.gui.getSelectedUnit()
     if not unit then return end
-    local helpText = customRawTokens.getToken(unit, "DFHACK_HELP_TEXT")
+    local helpText = customRawTokens.getToken(unit, "EXAMPLE_MOD_HELP_TEXT")
     if helpText then print(helpText) end
 
 * Healing armour::
@@ -3268,7 +3378,7 @@ Examples:
     local healAmount = 0
     for _, entry in ipairs(unit.inventory) do
         if entry.mode == 2 then -- Worn
-            healAmount = healAmount + tonumber((customRawTokens.getToken(entry.item, "DFHACK_HEAL_AMOUNT")) or 0)
+            healAmount = healAmount + tonumber((customRawTokens.getToken(entry.item, "EXAMPLE_MOD_HEAL_AMOUNT")) or 0)
         end
     end
     unit.body.blood_count = math.min(unit.body.blood_max, unit.body.blood_count + healAmount)
@@ -3869,6 +3979,7 @@ Attributes:
           If it returns false, the character is ignored.
 :on_change: Change notification callback; used as ``on_change(new_text,old_text)``.
 :on_submit: Enter key callback; if set the field will handle the key and call ``on_submit(text)``.
+:on_submit2: Shift-Enter key callback; if set the field will handle the key and call ``on_submit2(text)``.
 :key: If specified, the field is disabled until this key is pressed. Must be given as a string.
 :key_sep: If specified, will be used to customize how the activation key is
           displayed. See ``token.key_sep`` in the ``Label`` documentation below.
@@ -3890,6 +4001,14 @@ and then call the ``on_submit`` callback. Pressing the Escape key will also
 release keyboard focus, but first it will restore the text that was displayed
 before the ``EditField`` gained focus and then call the ``on_change`` callback.
 
+The ``EditField`` cursor can be moved to where you want to insert/remove text.
+You can click where you want the cursor to move or you can use any of the
+following keyboard hotkeys:
+
+- Left/Right arrow: move the cursor one character to the left or right.
+- Ctrl-Left/Right arrow: move the cursor one word to the left or right.
+- Alt-Left/Right arrow: move the cursor to the beginning/end of the text.
+
 Label class
 -----------
 
@@ -3906,7 +4025,10 @@ It has the following attributes:
 :auto_width: Sets self.frame.w from the text width.
 :on_click: A callback called when the label is clicked (optional)
 :on_rclick: A callback called when the label is right-clicked (optional)
-:scroll_keys: Specifies which keys the label should react to as a table. Default is ``STANDARDSCROLL`` (up or down arrows, page up or down).
+:scroll_keys: Specifies which keys the label should react to as a table. The table should map
+    keys to the number of lines to scroll as positive or negative integers or one of the keywords
+    supported by the ``scroll`` method. The default is up/down arrows scrolling by one line and page
+    up/down scrolling by one page.
 :show_scroll_icons: Controls scroll icons' behaviour: ``false`` for no icons, ``'right'`` or ``'left'`` for
     icons next to the text in an additional column (``frame_inset`` is adjusted to have ``.r`` or ``.l`` greater than ``0``),
     ``nil`` same as ``'right'`` but changes ``frame_inset`` only if a scroll icon is actually necessary
@@ -4002,6 +4124,12 @@ The Label widget implements the following methods:
 
   Computes the width of the text.
 
+* ``label:scroll(nlines)``
+
+  This method takes the number of lines to scroll as positive or negative
+  integers or one of the following keywords: ``+page``, ``-page``,
+  ``+halfpage``, or ``-halfpage``.
+
 WrappedLabel class
 ------------------
 
@@ -4044,7 +4172,7 @@ HotkeyLabel class
 -----------------
 
 This Label subclass is a convenience class for formatting text that responds to
-a hotkey.
+a hotkey or mouse click.
 
 It has the following attributes:
 
@@ -4054,13 +4182,13 @@ It has the following attributes:
 :label: The string (or a function that returns a string) to display after the
     hotkey.
 :on_activate: If specified, it is the callback that will be called whenever
-    the hotkey is pressed.
+    the hotkey is pressed or the label is clicked.
 
 CycleHotkeyLabel class
 ----------------------
 
 This Label subclass represents a group of related options that the user can
-cycle through by pressing a specified hotkey.
+cycle through by pressing a specified hotkey or clicking on the text.
 
 It has the following attributes:
 
@@ -4103,7 +4231,8 @@ This is a specialized subclass of CycleHotkeyLabel that has two options:
 List class
 ----------
 
-The List widget implements a simple list with paging.
+The List widget implements a simple list with paging. You can click on a list
+item to call the ``on_submit`` callback for that item.
 
 It has the following attributes:
 
@@ -4114,8 +4243,8 @@ It has the following attributes:
 :on_select: Selection change callback; called as ``on_select(index,choice)``.
             This is also called with *nil* arguments if ``setChoices`` is called
             with an empty list.
-:on_submit: Enter key callback; if specified, the list reacts to the key
+:on_submit: Enter key or mouse click callback; if specified, the list reacts to the
-            and calls it as ``on_submit(index,choice)``.
+            key/click and calls the callback as ``on_submit(index,choice)``.
 :on_submit2: Shift-Enter key callback; if specified, the list reacts to the key
              and calls it as ``on_submit2(index,choice)``.
 :row_height: Height of every row in text lines.

docs/Plugins.rst

@@ -373,29 +373,55 @@ selected objects.
 
 prospect
 ========
-Prints a big list of all the present minerals and plants. By default, only
-the visible part of the map is scanned.
 
-Options:
+**Usage:**
 
-:all:   Scan the whole map, as if it were revealed.
+    ``prospect [all|hell] [<options>]``
-:value: Show material value in the output. Most useful for gems.
-:hell:  Show the Z range of HFS tubes. Implies 'all'.
 
-If prospect is called during the embark selection screen, it displays an estimate of
+Shows a summary of resources that exist on the map. By default, only the visible
-layer stone availability.
+part of the map is scanned. Include the ``all`` keyword if you want ``prospect``
+to scan the whole map as if it were revealed. Use ``hell`` instead of ``all`` if
+you also want to see the Z range of HFS tubes in the 'features' report section.
 
-.. note::
+**Options:**
+
+:``-h``, ``--help``:
+    Shows this help text.
+:``-s``, ``--show <sections>``:
+    Shows only the named comma-separated list of report sections. Report section
+    names are: summary, liquids, layers, features, ores, gems, veins, shrubs,
+    and trees. If run during pre-embark, only the layers, ores, gems, and veins
+    report sections are available.
+:``-v``, ``--values``:
+    Includes material value in the output. Most useful for the 'gems' report
+    section.
 
-    The results of pre-embark prospect are an *estimate*, and can at best be expected
+**Examples:**
-    to be somewhere within +/- 30% of the true amount; sometimes it does a lot worse.
-    Especially, it is not clear how to precisely compute how many soil layers there
-    will be in a given embark tile, so it can report a whole extra layer, or omit one
-    that is actually present.
 
-Options:
+``prospect all``
+    Shows the entire report for the entire map.
+
+``prospect hell --show layers,ores,veins``
+    Shows only the layers, ores, and other vein stone report sections, and
+    includes information on HFS tubes when a fort is loaded.
+
+``prospect all -sores``
+    Show only information about ores for the pre-embark or fortress map report.
 
-:all:    Also estimate vein mineral amounts.
+**Pre-embark estimate:**
+
+If prospect is called during the embark selection screen, it displays an
+estimate of layer stone availability. If the ``all`` keyword is specified, it
+also estimates ores, gems, and vein material. The estimate covers all tiles of
+the embark rectangle.
+
+.. note::
+
+    The results of pre-embark prospect are an *estimate*, and can at best be
+    expected to be somewhere within +/- 30% of the true amount; sometimes it
+    does a lot worse. Especially, it is not clear how to precisely compute how
+    many soil layers there will be in a given embark tile, so it can report a
+    whole extra layer, or omit one that is actually present.
 
 .. _remotefortressreader:
 
@@ -1103,6 +1129,7 @@ A plugin for manipulating manager orders.
 
 Subcommands:
 
+:list: Shows the list of previously exported orders, including the orders library.
 :export NAME: Exports the current list of manager orders to a file named ``dfhack-config/orders/NAME.json``.
 :import NAME: Imports manager orders from a file named ``dfhack-config/orders/NAME.json``.
 :clear: Deletes all manager orders in the current embark.
@@ -1115,6 +1142,103 @@ your ``onMapLoad.init`` file::
 
     repeat -name orders-sort -time 1 -timeUnits days -command [ orders sort ]
 
+
+The orders library
+------------------
+
+DFHack comes with a library of useful manager orders that are ready for import:
+
+:source:`basic.json <data/orders/basic.json>`
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This collection of orders handles basic fort necessities:
+
+- prepared meals and food products (and by-products like oil)
+- booze/mead
+- thread/cloth/dye
+- pots/jugs/buckets/mugs
+- bags of leather, cloth, silk, and yarn
+- crafts and totems from otherwise unusable by-products
+- mechanisms/cages
+- splints/crutches
+- lye/soap
+- ash/potash
+- beds/wheelbarrows/minecarts
+- scrolls
+
+You should import it as soon as you have enough dwarves to perform the tasks.
+Right after the first migration wave is usually a good time.
+
+:source:`furnace.json <data/orders/furnace.json>`
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This collection creates basic items that require heat. It is separated out from
+``basic.json`` to give players the opportunity to set up magma furnaces first in
+order to save resources. It handles:
+
+- charcoal (including smelting of bituminous coal and lignite)
+- pearlash
+- sand
+- green/clear/crystal glass
+- adamantine processing
+- item melting
+
+Orders are missing for plaster powder until DF :bug:`11803` is fixed.
+
+:source:`military.json <data/orders/military.json>`
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This collection adds high-volume smelting jobs for military-grade metal ores and
+produces weapons and armor:
+
+- leather backpacks/waterskins/cloaks/quivers/armor
+- bone/wooden bolts
+- smelting for platinum, silver, steel, bronze, bismuth bronze, and copper (and
+  their dependencies)
+- bronze/bismuth bronze/copper bolts
+- platinum/silver/steel/iron/bismuth bronze/bronze/copper weapons and armor,
+  with checks to ensure only the best available materials are being used
+
+If you set a stockpile to take weapons and armor of less than masterwork quality
+and turn on `automelt` (like what `dreamfort` provides on its industry level),
+these orders will automatically upgrade your military equipment to masterwork.
+Make sure you have a lot of fuel (or magma forges and furnaces) before you turn
+``automelt`` on, though!
+
+This file should only be imported, of course, if you need to equip a military.
+
+:source:`smelting.json <data/orders/smelting.json>`
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This collection adds smelting jobs for all ores. It includes handling the ores
+already managed by ``military.json``, but has lower limits. This ensures all
+ores will be covered if a player imports ``smelting`` but not ``military``, but
+the higher-volume ``military`` orders will take priority if both are imported.
+
+:source:`rockstock.json <data/orders/rockstock.json>`
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This collection of orders keeps a small stock of all types of rock furniture.
+This allows you to do ad-hoc furnishings of guildhalls, libraries, temples, or
+other rooms with `buildingplan` and your masons will make sure there is always
+stock on hand to fulfill the plans.
+
+:source:`glassstock.json <data/orders/glassstock.json>`
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Similar to ``rockstock`` above, this collection keeps a small stock of all types
+of glass furniture. If you have a functioning glass industry, this is more
+sustainable than ``rockstock`` since you can never run out of sand. If you have
+plenty of rock and just want the variety, you can import both ``rockstock`` and
+``glassstock`` to get a mixture of rock and glass furnishings in your fort.
+
+There are a few items that ``glassstock`` produces that ``rockstock`` does not,
+since there are some items that can not be made out of rock, for example:
+
+- tubes and corkscrews for building magma-safe screw pumps
+- windows
+- terrariums (as an alternative to wooden cages)
+
 .. _seedwatch:
 
 seedwatch
@@ -2972,8 +3096,121 @@ To apply a profession, either highlight a single dwarf or select multiple with
 :kbd:`x`, and press :kbd:`p` to select the profession to apply. All labors for
 the selected dwarves will be reset to the labors of the chosen profession.
 
-Professions are saved as human-readable text files in the "professions" folder
+Professions are saved as human-readable text files in the
-within the DF folder, and can be edited or deleted there.
+``dfhack-config/professions`` folder within the DF folder, and can be edited or
+deleted there.
+
+The professions library
+~~~~~~~~~~~~~~~~~~~~~~~
+
+The manipulator plugin comes with a library of professions that you can assign
+to your dwarves.
+
+If you'd rather use Dwarf Therapist to manage your labors, it is easy to import
+these professions to DT and use them there. Simply assign the professions you
+want to import to a dwarf. Once you have assigned a profession to at least one
+dwarf, you can select "Import Professions from DF" in the DT "File" menu. The
+professions will then be available for use in DT.
+
+In the charts below, the "At Start" and "Max" columns indicate the approximate
+number of dwarves of each profession that you are likely to need at the start of
+the game and how many you are likely to need in a mature fort. These are just
+approximations. Your playstyle may demand more or fewer of each profession.
+
+=============  ========  ===== =================================================
+Profession     At Start  Max   Description
+=============  ========  ===== =================================================
+Chef           0         3     Buchery, Tanning, and Cooking. It is important to
+                               focus just a few dwarves on cooking since
+                               well-crafted meals make dwarves very happy. They
+                               are also an excellent trade good.
+Craftsdwarf    0         4-6   All labors used at Craftsdwarf's workshops,
+                               Glassmaker's workshops, and kilns.
+Doctor         0         2-4   The full suite of medical labors, plus Animal
+                               Caretaking for those using the dwarfvet plugin.
+Farmer         1         4     Food- and animal product-related labors. This
+                               profession also has the ``Alchemist`` labor
+                               enabled since they need to focus on food-related
+                               jobs, though you might want to disable
+                               ``Alchemist`` for your first farmer until there
+                               are actual farming duties to perform.
+Fisherdwarf    0         0-1   Fishing and fish cleaning. If you assign this
+                               profession to any dwarf, be prepared to be
+                               inundated with fish. Fisherdwarves *never stop
+                               fishing*. Be sure to also run ``prioritize -a
+                               PrepareRawFish ExtractFromRawFish`` or else
+                               caught fish will just be left to rot.
+Hauler         0         >20   All hauling labors plus Siege Operating, Mechanic
+                               (so haulers can assist in reloading traps) and
+                               Architecture (so haulers can help build massive
+                               windmill farms and pump stacks). As you
+                               accumulate enough Haulers, you can turn off
+                               hauling labors for other dwarves so they can
+                               focus on their skilled tasks. You may also want
+                               to restrict your Mechanic's workshops to only
+                               skilled mechanics so your haulers don't make
+                               low-quality mechanisms.
+Laborer        0         10-12 All labors that don't improve quality with skill,
+                               such as Soapmaking and furnace labors.
+Marksdwarf     0         10-30 Similar to Hauler. See the description for
+                               Meleedwarf below for more details.
+Mason          2         2-4   Masonry and Gem Cutting/Encrusting. In the early
+                               game, you may need to run "`prioritize`
+                               ConstructBuilding" to get your masons to build
+                               wells and bridges if they are too busy crafting
+                               stone furniture.
+Meleedwarf     0         20-50 Similar to Hauler, but without most civilian
+                               labors. This profession is separate from Hauler
+                               so you can find your military dwarves easily.
+                               Meleedwarves and Marksdwarves have Mechanics and
+                               hauling labors enabled so you can temporarily
+                               deactivate your military after sieges and allow
+                               your military dwarves to help clean up.
+Migrant        0         0     You can assign this profession to new migrants
+                               temporarily while you sort them into professions.
+                               Like Marksdwarf and Meleedwarf, the purpose of
+                               this profession is so you can find your new
+                               dwarves more easily.
+Miner          2         2-10  Mining and Engraving. This profession also has
+                               the ``Alchemist`` labor enabled, which disables
+                               hauling for those using the `autohauler` plugin.
+                               Once the need for Miners tapers off in the late
+                               game, dwarves with this profession make good
+                               military dwarves, wielding their picks as
+                               weapons.
+Outdoorsdwarf  1         2-4   Carpentry, Bowyery, Woodcutting, Animal Training,
+                               Trapping, Plant Gathering, Beekeeping, and Siege
+                               Engineering.
+Smith          0         2-4   Smithing labors. You may want to specialize your
+                               Smiths to focus on a single smithing skill to
+                               maximize equipment quality.
+StartManager   1         0     All skills not covered by the other starting
+                               professions (Miner, Mason, Outdoorsdwarf, and
+                               Farmer), plus a few overlapping skills to
+                               assist in critical tasks at the beginning of the
+                               game. Individual labors should be turned off as
+                               migrants are assigned more specialized
+                               professions that cover them, and the StartManager
+                               dwarf can eventually convert to some other
+                               profession.
+Tailor         0         2     Textile industry labors: Dying, Leatherworking,
+                               Weaving, and Clothesmaking.
+=============  ========  ===== =================================================
+
+A note on autohauler
+~~~~~~~~~~~~~~~~~~~~
+
+These profession definitions are designed to work well with or without the
+`autohauler` plugin (which helps to keep your dwarves focused on skilled labors
+instead of constantly being distracted by hauling). If you do want to use
+autohauler, adding the following lines to your ``onMapLoad.init`` file will
+configure it to let the professions manage the "Feed water to civilians" and
+"Recover wounded" labors instead of enabling those labors for all hauling
+dwarves::
+
+    on-new-fortress enable autohauler
+    on-new-fortress autohauler FEED_WATER_CIVILIANS allow
+    on-new-fortress autohauler RECOVER_WOUNDED allow
 
 .. _mousequery:
 

docs/changelog.txt

@@ -34,19 +34,48 @@ changelog.txt uses a syntax similar to RST, with a few special sequences:
 # Future
 
 ## New Plugins
+- `autonestbox`: split off from `zone` into its own plugin. Note that to enable, the command has changed from ``autonestbox start`` to ``enable autonestbox``.
+- `autobutcher`: split off from `zone` into its own plugin. Note that to enable, the command has changed from ``autobutcher start`` to ``enable autobutcher``.
 
 ## New Tweaks
 
 ## Fixes
+- ``job.removeJob()``: ensure jobs are removed from the world list when they are canceled
 
 ## Misc Improvements
+- Init scripts: ``dfhack.init`` and other init scripts have moved to ``dfhack-config/init/``. If you have customized your ``dfhack.init`` file and want to keep your changes, please move the part that you have customized to the new location at ``dfhack-config/init/dfhack.init``. If you do not have changes that you want to keep, do not copy anything, and the new defaults will be used automatically.
+- History files: ``dfhack.history``, ``tiletypes.history``, ``lua.history``, and ``liquids.history`` have moved to the ``dfhack-config`` directory. If you'd like to keep the contents of your current history files, please move them to ``dfhack-config``.
+- `do-job-now`: new global keybinding for boosting the priority of the jobs associated with the selected building/work order/unit/item etc.: Alt-N
+- `keybinding`: support backquote (\`) as a hotkey (and assign the hotkey to the new `gui/launcher` interface)
+- `manipulator`: add a library of useful default professions
+- `manipulator`: move professions configuration from ``professions/`` to ``dfhack-config/professions/`` to keep it together with other dfhack configuration. If you have saved professions that you would like to keep, please manually move them to the new folder.
+- ``materials.ItemTraitsDialog``: added a default ``on_select``-handler which toggles the traits.
+- `orders`: added useful library of manager orders. see them with ``orders list`` and import them with, for example, ``orders import library/basic``
+- `prospect`: add new ``--show`` option to give the player control over which report sections are shown. e.g. ``prospect all --show ores`` will just show information on ores.
+- `seedwatch`: ``seedwatch all`` now adds all plants with seeds to the watchlist, not just the "basic" crops.
+- UX: You can now move the cursor around in DFHack text fields in ``gui/`` scripts (e.g. `gui/blueprint`, `gui/quickfort`, or `gui/gm-editor`). You can move the cursor by clicking where you want it to go with the mouse or using the Left/Right arrow keys.  Ctrl+Left/Right will move one word at a time, and Alt+Left/Right will move to the beginning/end of the text.
+- UX: You can now click on the hotkey hint text in many ``gui/`` script windows to activate the hotkey, like a button. Not all scripts have been updated to use the clickable widget yet, but you can try it in `gui/blueprint` or `gui/quickfort`.
 
 ## Documentation
 - Added `modding-guide`
 
 ## API
+- Removed "egg" ("eggy") hook support (Linux only). The only remaining method of hooking into DF is by interposing SDL calls, which has been the method used by all binary releases of DFHack.
+- Removed ``Engravings`` module (C++-only). Access ``world.engravings`` directly instead.
+- Removed ``Notes`` module (C++-only). Access ``ui.waypoints.points`` directly instead.
+- Removed ``Windows`` module (C++-only) - unused.
+- ``Constructions`` module (C++-only): removed ``t_construction``, ``isValid()``, ``getCount()``, ``getConstruction()``, and ``copyConstruction()``. Access ``world.constructions`` directly instead.
+- ``Gui::getSelectedItem()``, ``Gui::getAnyItem()``: added support for the artifacts screen
 
 ## Lua
+- History: added ``dfhack.getCommandHistory(history_id, history_filename)`` and ``dfhack.addCommandToHistory(history_id, history_filename, command)`` so gui scripts can access a commandline history without requiring a terminal.
+- ``helpdb``: database and query interface for DFHack tool help text
+- ``tile-material``: fix the order of declarations. The ``GetTileMat`` function now returns the material as intended (always returned nil before). Also changed the license info, with permission of the original author.
+- ``widgets.EditField``: new ``onsubmit2`` callback attribute is called when the user hits Shift-Enter.
+- ``widgets.EditField``: new function: ``setCursor(position)`` sets the input cursor.
+- ``widgets.Label``: ``scroll`` function now interprets the keywords ``+page``, ``-page``, ``+halfpage``, and ``-halfpage`` in addition to simple positive and negative numbers.
+- ``widgets.HotkeyLabel``: clicking on the widget will now call ``on_activate()``.
+- ``widgets.CycleHotkeyLabel``: clicking on the widget will now cycle the options and trigger ``on_change()``. This also applies to the ``ToggleHotkeyLabel`` subclass.
 
 # 0.47.05-r6
 

docs/guides/examples-guide.rst

@@ -47,224 +47,3 @@ it is useful (and customizable) for any fort. It includes the following config:
   fortress is first started, so any later changes you make to autobutcher
   settings won't be overridden.
 - Enables `automelt`, `tailor`, `zone`, `nestboxes`, and `autonestbox`.
-
-The ``orders/`` subfolder
--------------------------
-
-The :source:`orders/ <data/examples/orders>` subfolder contains manager orders
-that, along with the ``onMapLoad_dreamfort.init`` file above, allow a fort to be
-self-sustaining. Copy them to your ``dfhack-config/orders/`` folder and import
-as required with the `orders` DFHack plugin.
-
-:source:`basic.json <data/examples/orders/basic.json>`
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-This collection of orders handles basic fort necessities:
-
-- prepared meals and food products (and by-products like oil)
-- booze/mead
-- thread/cloth/dye
-- pots/jugs/buckets/mugs
-- bags of leather, cloth, silk, and yarn
-- crafts and totems from otherwise unusable by-products
-- mechanisms/cages
-- splints/crutches
-- lye/soap
-- ash/potash
-- beds/wheelbarrows/minecarts
-- scrolls
-
-You should import it as soon as you have enough dwarves to perform the tasks.
-Right after the first migration wave is usually a good time.
-
-:source:`furnace.json <data/examples/orders/furnace.json>`
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-This collection creates basic items that require heat. It is separated out from
-``basic.json`` to give players the opportunity to set up magma furnaces first in
-order to save resources. It handles:
-
-- charcoal (including smelting of bituminous coal and lignite)
-- pearlash
-- sand
-- green/clear/crystal glass
-- adamantine processing
-- item melting
-
-Orders are missing for plaster powder until DF :bug:`11803` is fixed.
-
-:source:`military.json <data/examples/orders/military.json>`
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-This collection adds high-volume smelting jobs for military-grade metal ores and
-produces weapons and armor:
-
-- leather backpacks/waterskins/cloaks/quivers/armor
-- bone/wooden bolts
-- smelting for platinum, silver, steel, bronze, bismuth bronze, and copper (and
-  their dependencies)
-- bronze/bismuth bronze/copper bolts
-- platinum/silver/steel/iron/bismuth bronze/bronze/copper weapons and armor,
-  with checks to ensure only the best available materials are being used
-
-If you set a stockpile to take weapons and armor of less than masterwork quality
-and turn on `automelt` (like what `dreamfort` provides on its industry level),
-these orders will automatically upgrade your military equipment to masterwork.
-Make sure you have a lot of fuel (or magma forges and furnaces) before you turn
-``automelt`` on, though!
-
-This file should only be imported, of course, if you need to equip a military.
-
-:source:`smelting.json <data/examples/orders/smelting.json>`
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-This collection adds smelting jobs for all ores. It includes handling the ores
-already managed by ``military.json``, but has lower limits. This ensures all
-ores will be covered if a player imports ``smelting`` but not ``military``, but
-the higher-volume ``military`` orders will take priority if both are imported.
-
-:source:`rockstock.json <data/examples/orders/rockstock.json>`
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-This collection of orders keeps a small stock of all types of rock furniture.
-This allows you to do ad-hoc furnishings of guildhalls, libraries, temples, or
-other rooms with `buildingplan` and your masons will make sure there is always
-stock on hand to fulfill the plans.
-
-:source:`glassstock.json <data/examples/orders/glassstock.json>`
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Similar to ``rockstock`` above, this collection keeps a small stock of all types
-of glass furniture. If you have a functioning glass industry, this is more
-sustainable than ``rockstock`` since you can never run out of sand. If you have
-plenty of rock and just want the variety, you can import both ``rockstock`` and
-``glassstock`` to get a mixture of rock and glass furnishings in your fort.
-
-There are a few items that ``glassstock`` produces that ``rockstock`` does not,
-since there are some items that can not be made out of rock, for example:
-
-- tubes and corkscrews for building magma-safe screw pumps
-- windows
-- terrariums (as an alternative to wooden cages)
-
-The ``professions/`` subfolder
-------------------------------
-
-The :source:`professions/ <data/examples/professions>` subfolder contains
-professions, or sets of related labors, that you can assign to your dwarves with
-the DFHack `manipulator` plugin. Copy them into the ``professions/``
-subdirectory under the main Dwarf Fortress folder (you may have to create this
-subdirectory) and assign them to your dwarves in the manipulator UI, accessible
-from the ``units`` screen via the :kbd:`l` hotkey. Make sure that the
-``manipulator`` plugin is enabled in your ``dfhack.init`` file! You can assign a
-profession to a dwarf by selecting the dwarf in the ``manipulator`` UI and
-hitting :kbd:`p`. The list of professions that you copied into the
-``professions/`` folder will show up for you to choose from. This is very useful
-for assigning roles to new migrants to ensure that all the tasks in your fort
-have adequate numbers of dwarves attending to them.
-
-If you'd rather use Dwarf Therapist to manage your labors, it is easy to import
-these professions to DT and use them there. Simply assign the professions you
-want to import to a dwarf. Once you have assigned a profession to at least one
-dwarf, you can select "Import Professions from DF" in the DT "File" menu. The
-professions will then be available for use in DT.
-
-In the charts below, the "At Start" and "Max" columns indicate the approximate
-number of dwarves of each profession that you are likely to need at the start of
-the game and how many you are likely to need in a mature fort.
-
-=============  ========  ===== =================================================
-Profession     At Start  Max   Description
-=============  ========  ===== =================================================
-Chef           0         3     Buchery, Tanning, and Cooking. It is important to
-                               focus just a few dwarves on cooking since
-                               well-crafted meals make dwarves very happy. They
-                               are also an excellent trade good.
-Craftsdwarf    0         4-6   All labors used at Craftsdwarf's workshops,
-                               Glassmaker's workshops, and kilns.
-Doctor         0         2-4   The full suite of medical labors, plus Animal
-                               Caretaking for those using the dwarfvet plugin.
-Farmer         1         4     Food- and animal product-related labors. This
-                               profession also has the ``Alchemist`` labor
-                               enabled since they need to focus on food-related
-                               jobs, though you might want to disable
-                               ``Alchemist`` for your first farmer until there
-                               are actual farming duties to perform.
-Fisherdwarf    0         0-1   Fishing and fish cleaning. If you assign this
-                               profession to any dwarf, be prepared to be
-                               inundated with fish. Fisherdwarves *never stop
-                               fishing*. Be sure to also run ``prioritize -a
-                               PrepareRawFish ExtractFromRawFish`` (or use the
-                               ``onMapLoad_dreamfort.init`` file above) or else
-                               caught fish will just be left to rot.
-Hauler         0         >20   All hauling labors plus Siege Operating, Mechanic
-                               (so haulers can assist in reloading traps) and
-                               Architecture (so haulers can help build massive
-                               windmill farms and pump stacks). As you
-                               accumulate enough Haulers, you can turn off
-                               hauling labors for other dwarves so they can
-                               focus on their skilled tasks. You may also want
-                               to restrict your Mechanic's workshops to only
-                               skilled mechanics so your haulers don't make
-                               low-quality mechanisms.
-Laborer        0         10-12 All labors that don't improve quality with skill,
-                               such as Soapmaking and furnace labors.
-Marksdwarf     0         10-30 Similar to Hauler. See the description for
-                               Meleedwarf below for more details.
-Mason          2         2-4   Masonry, Gem Cutting/Encrusting, and
-                               Architecture. In the early game, you may need to
-                               run "`prioritize` ConstructBuilding" to get your
-                               masons to build wells and bridges if they are too
-                               busy crafting stone furniture.
-Meleedwarf     0         20-50 Similar to Hauler, but without most civilian
-                               labors. This profession is separate from Hauler
-                               so you can find your military dwarves easily.
-                               Meleedwarves and Marksdwarves have Mechanics and
-                               hauling labors enabled so you can temporarily
-                               deactivate your military after sieges and allow
-                               your military dwarves to help clean up.
-Migrant        0         0     You can assign this profession to new migrants
-                               temporarily while you sort them into professions.
-                               Like Marksdwarf and Meleedwarf, the purpose of
-                               this profession is so you can find your new
-                               dwarves more easily.
-Miner          2         2-10  Mining and Engraving. This profession also has
-                               the ``Alchemist`` labor enabled, which disables
-                               hauling for those using the `autohauler` plugin.
-                               Once the need for Miners tapers off in the late
-                               game, dwarves with this profession make good
-                               military dwarves, wielding their picks as
-                               weapons.
-Outdoorsdwarf  1         2-4   Carpentry, Bowyery, Woodcutting, Animal Training,
-                               Trapping, Plant Gathering, Beekeeping, and Siege
-                               Engineering.
-Smith          0         2-4   Smithing labors. You may want to specialize your
-                               Smiths to focus on a single smithing skill to
-                               maximize equipment quality.
-StartManager   1         0     All skills not covered by the other starting
-                               professions (Miner, Mason, Outdoorsdwarf, and
-                               Farmer), plus a few overlapping skills to
-                               assist in critical tasks at the beginning of the
-                               game. Individual labors should be turned off as
-                               migrants are assigned more specialized
-                               professions that cover them, and the StartManager
-                               dwarf can eventually convert to some other
-                               profession.
-Tailor         0         2     Textile industry labors: Dying, Leatherworking,
-                               Weaving, and Clothesmaking.
-=============  ========  ===== =================================================
-
-A note on autohauler
-~~~~~~~~~~~~~~~~~~~~
-
-These profession definitions are designed to work well with or without the
-`autohauler` plugin (which helps to keep your dwarves focused on skilled labors
-instead of constantly being distracted by hauling). If you do want to use
-autohauler, adding the following lines to your ``onMapLoad.init`` file will
-configure it to let the professions manage the "Feed water to civilians" and
-"Recover wounded" labors instead of enabling those labors for all hauling
-dwarves::
-
-    on-new-fortress enable autohauler
-    on-new-fortress autohauler FEED_WATER_CIVILIANS allow
-    on-new-fortress autohauler RECOVER_WOUNDED allow

library/CMakeLists.txt

@@ -114,19 +114,11 @@ set(MAIN_SOURCES_DARWIN
     Process-darwin.cpp
 )
 
-set(MAIN_SOURCES_LINUX_EGGY
-    ${CONSOLE_SOURCES}
-    Hooks-egg.cpp
-    PlugLoad-linux.cpp
-    Process-linux.cpp
-)
-
 set(MODULE_HEADERS
     include/modules/Buildings.h
     include/modules/Burrows.h
     include/modules/Constructions.h
     include/modules/Designations.h
-    include/modules/Engravings.h
     include/modules/EventManager.h
     include/modules/Filesystem.h
     include/modules/Graphic.h
@@ -138,7 +130,6 @@ set(MODULE_HEADERS
     include/modules/MapCache.h
     include/modules/Maps.h
     include/modules/Materials.h
-    include/modules/Notes.h
     include/modules/Once.h
     include/modules/Persistence.h
     include/modules/Random.h
@@ -154,7 +145,6 @@ set(MODULE_SOURCES
     modules/Burrows.cpp
     modules/Constructions.cpp
     modules/Designations.cpp
-    modules/Engravings.cpp
     modules/EventManager.cpp
     modules/Filesystem.cpp
     modules/Graphic.cpp
@@ -165,7 +155,6 @@ set(MODULE_SOURCES
     modules/MapCache.cpp
     modules/Maps.cpp
     modules/Materials.cpp
-    modules/Notes.cpp
     modules/Once.cpp
     modules/Persistence.cpp
     modules/Random.cpp
@@ -173,7 +162,6 @@ set(MODULE_SOURCES
     modules/Screen.cpp
     modules/Translation.cpp
     modules/Units.cpp
-    modules/Windows.cpp
     modules/World.cpp
 )
 
@@ -211,10 +199,7 @@ list(APPEND PROJECT_SOURCES ${MAIN_SOURCES})
 list(APPEND PROJECT_SOURCES ${MODULE_SOURCES})
 
 if(UNIX)
-    option(BUILD_EGGY "Make DFHack strangely egg-shaped." OFF)
+    if(APPLE)
-    if(BUILD_EGGY)
-        list(APPEND PROJECT_SOURCES ${MAIN_SOURCES_LINUX_EGGY})
-    elseif(APPLE)
         list(APPEND PROJECT_SOURCES ${MAIN_SOURCES_DARWIN})
     else()
         list(APPEND PROJECT_SOURCES ${MAIN_SOURCES_LINUX})
@@ -376,15 +361,9 @@ add_executable(dfhack-run dfhack-run.cpp)
 add_executable(binpatch binpatch.cpp)
 target_link_libraries(binpatch dfhack-md5)
 
-if(BUILD_EGGY)
-    set_target_properties(dfhack PROPERTIES OUTPUT_NAME "egg" )
-else()
-    if(WIN32)
-        set_target_properties(dfhack PROPERTIES OUTPUT_NAME "SDL" )
-    endif()
-endif()
-
 if(WIN32)
+    # name the resulting library SDL.dll on Windows
+    set_target_properties(dfhack PROPERTIES OUTPUT_NAME "SDL" )
     set_target_properties(dfhack PROPERTIES COMPILE_FLAGS "/FI\"Export.h\"" )
     set_target_properties(dfhack-client PROPERTIES COMPILE_FLAGS "/FI\"Export.h\"" )
 else()
@@ -439,30 +418,19 @@ if(UNIX)
             DESTINATION .)
     endif()
 else()
-    if(NOT BUILD_EGGY)
     # On windows, copy the renamed SDL so DF can still run.
     install(PROGRAMS ${dfhack_SOURCE_DIR}/package/windows/win${DFHACK_BUILD_ARCH}/SDLreal.dll
         DESTINATION ${DFHACK_LIBRARY_DESTINATION})
-    endif()
 endif()
 
 # install the main lib
-if(NOT BUILD_EGGY)
+install(TARGETS dfhack
-    install(TARGETS dfhack
     LIBRARY DESTINATION ${DFHACK_LIBRARY_DESTINATION}
     RUNTIME DESTINATION ${DFHACK_LIBRARY_DESTINATION})
-else()
-    install(TARGETS dfhack
-        LIBRARY DESTINATION ${DFHACK_EGGY_DESTINATION}
-        RUNTIME DESTINATION ${DFHACK_EGGY_DESTINATION})
-endif()
 
 # install the offset file
 install(FILES xml/symbols.xml
     DESTINATION ${DFHACK_DATA_DESTINATION})
-# install the example autoexec file
-install(FILES ../dfhack.init-example
-    DESTINATION ${DFHACK_BINARY_DESTINATION})
 
 install(TARGETS dfhack-run dfhack-client binpatch
     LIBRARY DESTINATION ${DFHACK_LIBRARY_DESTINATION}

library/Core.cpp

@@ -53,8 +53,6 @@ using namespace std;
 #include "modules/Filesystem.h"
 #include "modules/Gui.h"
 #include "modules/World.h"
-#include "modules/Graphic.h"
-#include "modules/Windows.h"
 #include "modules/Persistence.h"
 #include "RemoteServer.h"
 #include "RemoteTools.h"
@@ -1450,13 +1448,12 @@ static void run_dfhack_init(color_ostream &out, Core *core)
         return;
     }
 
+    // load baseline defaults
+    core->loadScriptFile(out, "dfhack-config/init/default.dfhack.init", false);
+
+    // load user overrides
     std::vector<std::string> prefixes(1, "dfhack");
-    size_t count = loadScriptFiles(core, out, prefixes, ".");
+    loadScriptFiles(core, out, prefixes, "dfhack-config/init");
-    if (!count || !Filesystem::isfile("dfhack.init"))
-    {
-        core->runCommand(out, "gui/no-dfhack-init");
-        core->loadScriptFile(out, "dfhack.init-example", false);
-    }
 }
 
 // Load dfhack.init in a dedicated thread (non-interactive console mode)
@@ -1472,12 +1469,14 @@ void fInitthread(void * iodata)
 // A thread function... for the interactive console.
 void fIOthread(void * iodata)
 {
+    static const char * HISTORY_FILE = "dfhack-config/dfhack.history";
+
     IODATA * iod = ((IODATA*) iodata);
     Core * core = iod->core;
     PluginManager * plug_mgr = ((IODATA*) iodata)->plug_mgr;
 
     CommandHistory main_history;
-    main_history.load("dfhack.history");
+    main_history.load(HISTORY_FILE);
 
     Console & con = core->getConsole();
     if (plug_mgr == 0)
@@ -1518,7 +1517,7 @@ void fIOthread(void * iodata)
         {
             // a proper, non-empty command was entered
             main_history.add(command);
-            main_history.save("dfhack.history");
+            main_history.save(HISTORY_FILE);
         }
 
         auto rv = core->runCommand(con, command);
@@ -1564,7 +1563,6 @@ Core::Core() :
     last_local_map_ptr = NULL;
     last_pause_state = false;
     top_viewscreen = NULL;
-    screen_window = NULL;
 
     color_ostream::log_errors_to_stderr = true;
 
@@ -1835,8 +1833,6 @@ bool Core::Init()
     cerr << "Starting DF input capture thread.\n";
     // set up hotkey capture
     d->hotkeythread = std::thread(fHKthread, (void *) temp);
-    screen_window = new Windows::top_level_window();
-    screen_window->addChild(new Windows::dfhack_dummy(5,10));
     started = true;
     modstate = 0;
 
@@ -1980,14 +1976,6 @@ bool Core::isSuspended(void)
     return ownerThread.load() == std::this_thread::get_id();
 }
 
-int Core::TileUpdate()
-{
-    if(!started)
-        return false;
-    screen_window->paint();
-    return true;
-}
-
 void Core::doUpdate(color_ostream &out, bool first_update)
 {
     Lua::Core::Reset(out, "DF code execution");
@@ -2226,7 +2214,11 @@ void Core::handleLoadAndUnloadScripts(color_ostream& out, state_change_event eve
     auto i = table.find(event);
     if ( i != table.end() ) {
         const std::vector<std::string>& set = i->second;
-        loadScriptFiles(this, out, set, "."      );
+
+        // load baseline defaults
+        this->loadScriptFile(out, "dfhack-config/init/default." + set[0] + ".init", false);
+
+        loadScriptFiles(this, out, set, "dfhack-config/init");
         loadScriptFiles(this, out, set, rawFolder);
         loadScriptFiles(this, out, set, rawFolder + "objects/");
     }
@@ -2621,6 +2613,9 @@ static bool parseKeySpec(std::string keyspec, int *psym, int *pmod, std::string
     if (keyspec.size() == 1 && keyspec[0] >= 'A' && keyspec[0] <= 'Z') {
         *psym = SDL::K_a + (keyspec[0]-'A');
         return true;
+    } else if (keyspec.size() == 1 && keyspec[0] == '`') {
+        *psym = SDL::K_BACKQUOTE;
+        return true;
     } else if (keyspec.size() == 1 && keyspec[0] >= '0' && keyspec[0] <= '9') {
         *psym = SDL::K_0 + (keyspec[0]-'0');
         return true;
@@ -2922,5 +2917,4 @@ TYPE * Core::get##TYPE() \
 }
 
 MODULE_GETTER(Materials);
-MODULE_GETTER(Notes);
 MODULE_GETTER(Graphic);

library/Hooks-egg.cpp

@@ -1,92 +0,0 @@
-/*
-https://github.com/peterix/dfhack
-Copyright (c) 2009-2012 Petr Mrázek (peterix@gmail.com)
-
-This software is provided 'as-is', without any express or implied
-warranty. In no event will the authors be held liable for any
-damages arising from the use of this software.
-
-Permission is granted to anyone to use this software for any
-purpose, including commercial applications, and to alter it and
-redistribute it freely, subject to the following restrictions:
-
-1. The origin of this software must not be misrepresented; you must
-not claim that you wrote the original software. If you use this
-software in a product, an acknowledgment in the product documentation
-would be appreciated but is not required.
-
-2. Altered source versions must be plainly marked as such, and
-must not be misrepresented as being the original software.
-
-3. This notice may not be removed or altered from any source
-distribution.
-*/
-#include <stdio.h>
-#include <stdint.h>
-#include <stdlib.h>
-#include <string.h>
-#include <vector>
-#include <string>
-#include <map>
-
-#include "Core.h"
-#include "Hooks.h"
-#include <iostream>
-
-// hook - called before rendering
-DFhackCExport int egg_init(void)
-{
-    // reroute stderr
-    freopen("stderr.log", "w", stderr);
-    // we don't reroute stdout until  we figure out if this should be done at all
-    // See: Console-linux.cpp
-    fprintf(stderr,"dfhack: hooking successful\n");
-    return true;
-}
-
-// hook - called before rendering
-DFhackCExport int egg_shutdown(void)
-{
-    DFHack::Core & c = DFHack::Core::getInstance();
-    return c.Shutdown();
-}
-
-// hook - called for each game tick (or more often)
-DFhackCExport int egg_tick(void)
-{
-    DFHack::Core & c = DFHack::Core::getInstance();
-    return c.Update();
-}
-// hook - called before rendering
-DFhackCExport int egg_prerender(void)
-{
-    DFHack::Core & c = DFHack::Core::getInstance();
-    return c.TileUpdate();
-}
-
-// hook - called for each SDL event, returns 0 when the event has been consumed. 1 otherwise
-DFhackCExport int egg_sdl_event(SDL::Event* event)
-{
-    // if the event is valid, intercept
-    if( event != 0 )
-    {
-        DFHack::Core & c = DFHack::Core::getInstance();
-        return c.DFH_SDL_Event(event);
-    }
-    return true;
-}
-
-// return this if you want to kill the event.
-const int curses_error = -1;
-// hook - ncurses event, -1 signifies error.
-DFhackCExport int egg_curses_event(int orig_return)
-{
-    /*
-    if(orig_return != -1)
-    {
-        DFHack::Core & c = DFHack::Core::getInstance();
-        int out;
-        return c.ncurses_wgetch(orig_return,);
-    }*/
-    return orig_return;
-}

library/LuaApi.cpp

@@ -148,6 +148,16 @@ void Lua::Push(lua_State *state, df::coord2d pos)
     lua_setfield(state, -2, "y");
 }
 
+void Lua::GetVector(lua_State *state, std::vector<std::string> &pvec)
+{
+    lua_pushnil(state);   // first key
+    while (lua_next(state, 1) != 0)
+    {
+        pvec.push_back(lua_tostring(state, -1));
+        lua_pop(state, 1);  // remove value, leave key
+    }
+}
+
 int Lua::PushPosXYZ(lua_State *state, df::coord pos)
 {
     if (!pos.isValid())
@@ -1359,6 +1369,38 @@ static void OpenRandom(lua_State *state)
     lua_pop(state, 1);
 }
 
+
+/*********************************
+* Commandline history repository *
+**********************************/
+
+static std::map<std::string, CommandHistory> commandHistories;
+
+static CommandHistory * ensureCommandHistory(std::string id,
+                                             std::string src_file) {
+    if (!commandHistories.count(id)) {
+        commandHistories[id].load(src_file.c_str());
+    }
+    return &commandHistories[id];
+}
+
+static int getCommandHistory(lua_State *state)
+{
+    std::string id = lua_tostring(state, 1);
+    std::string src_file = lua_tostring(state, 2);
+    std::vector<std::string> entries;
+    ensureCommandHistory(id, src_file)->getEntries(entries);
+    Lua::PushVector(state, entries);
+    return 1;
+}
+
+static void addCommandToHistory(std::string id, std::string src_file,
+                                std::string command) {
+    CommandHistory *history = ensureCommandHistory(id, src_file);
+    history->add(command);
+    history->save(src_file.c_str());
+}
+
 /************************
  * Wrappers for C++ API *
  ************************/
@@ -1450,6 +1492,12 @@ static const LuaWrapper::FunctionReg dfhack_module[] = {
     WRAP_VERSION_FUNC(gitXmlMatch, git_xml_match),
     WRAP_VERSION_FUNC(isRelease, is_release),
     WRAP_VERSION_FUNC(isPrerelease, is_prerelease),
+    WRAP(addCommandToHistory),
+    { NULL, NULL }
+};
+
+static const luaL_Reg dfhack_funcs[] = {
+    { "getCommandHistory", getCommandHistory },
     { NULL, NULL }
 };
 
@@ -3023,6 +3071,99 @@ static int internal_findScript(lua_State *L)
     return 1;
 }
 
+static int internal_listPlugins(lua_State *L)
+{
+    auto plugins = Core::getInstance().getPluginManager();
+
+    int i = 1;
+    lua_newtable(L);
+    for (auto it = plugins->begin(); it != plugins->end(); ++it)
+    {
+        lua_pushinteger(L, i++);
+        lua_pushstring(L, it->first.c_str());
+        lua_settable(L, -3);
+    }
+    return 1;
+}
+
+static int internal_listCommands(lua_State *L)
+{
+    auto plugins = Core::getInstance().getPluginManager();
+
+    const char *name = luaL_checkstring(L, 1);
+
+    auto plugin = plugins->getPluginByName(name);
+    if (!plugin)
+    {
+        lua_pushnil(L);
+        return 1;
+    }
+
+    size_t num_commands = plugin->size();
+    lua_newtable(L);
+    for (size_t i = 0; i < num_commands; ++i)
+    {
+        lua_pushinteger(L, i + 1);
+        lua_pushstring(L, (*plugin)[i].name.c_str());
+        lua_settable(L, -3);
+    }
+    return 1;
+}
+
+static const PluginCommand * getPluginCommand(const char * command)
+{
+    auto plugins = Core::getInstance().getPluginManager();
+    auto plugin = plugins->getPluginByCommand(command);
+    if (!plugin)
+    {
+        return NULL;
+    }
+
+    size_t num_commands = plugin->size();
+    for (size_t i = 0; i < num_commands; ++i)
+    {
+        if ((*plugin)[i].name == command)
+            return &(*plugin)[i];
+    }
+
+    // not found (somehow)
+    return NULL;
+}
+
+static int internal_getCommandHelp(lua_State *L)
+{
+    const PluginCommand *pc = getPluginCommand(luaL_checkstring(L, 1));
+    if (!pc)
+    {
+        lua_pushnil(L);
+        return 1;
+    }
+
+    std::string help = pc->description;
+    if (help.size() && help[help.size()-1] != '.')
+        help += ".";
+    if (pc->usage.size())
+        help += "\n" + pc->usage;
+    lua_pushstring(L, help.c_str());
+    return 1;
+}
+
+static int internal_getCommandDescription(lua_State *L)
+{
+    const PluginCommand *pc = getPluginCommand(luaL_checkstring(L, 1));
+    if (!pc)
+    {
+        lua_pushnil(L);
+        return 1;
+    }
+
+    std::string help = pc->description;
+    if (help.size() && help[help.size()-1] != '.')
+        help += ".";
+    lua_pushstring(L, help.c_str());
+    return 1;
+}
+
 static int internal_threadid(lua_State *L)
 {
     std::stringstream ss;
@@ -3094,6 +3235,10 @@ static const luaL_Reg dfhack_internal_funcs[] = {
     { "removeScriptPath", internal_removeScriptPath },
     { "getScriptPaths", internal_getScriptPaths },
     { "findScript", internal_findScript },
+    { "listPlugins", internal_listPlugins },
+    { "listCommands", internal_listCommands },
+    { "getCommandHelp", internal_getCommandHelp },
+    { "getCommandDescription", internal_getCommandDescription },
     { "threadid", internal_threadid },
     { "md5File", internal_md5file },
     { NULL, NULL }
@@ -3113,6 +3258,7 @@ void OpenDFHackApi(lua_State *state)
     OpenRandom(state);
 
     LuaWrapper::SetFunctionWrappers(state, dfhack_module);
+    luaL_setfuncs(state, dfhack_funcs, 0);
     OpenModule(state, "gui", dfhack_gui_module, dfhack_gui_funcs);
     OpenModule(state, "job", dfhack_job_module, dfhack_job_funcs);
     OpenModule(state, "units", dfhack_units_module, dfhack_units_funcs);

library/LuaTools.cpp

@@ -1145,7 +1145,7 @@ bool DFHack::Lua::InterpreterLoop(color_ostream &out, lua_State *state,
         return false;
 
     if (!hfile)
-        hfile = "lua.history";
+        hfile = "dfhack-config/lua.history";
     if (!prompt)
         prompt = "lua";
 

library/PlugLoad-windows.cpp

@@ -35,7 +35,6 @@ distribution.
 #include <stdio.h>
 
 #include "tinythread.h"
-#include "modules/Graphic.h"
 #include "../plugins/uicommon.h"
 
 /*

library/include/Console.h

@@ -32,6 +32,7 @@ distribution.
 #include <assert.h>
 #include <iostream>
 #include <string>
+#include <vector>
 namespace tthread
 {
     class mutex;
@@ -44,7 +45,7 @@ namespace  DFHack
     class CommandHistory
     {
     public:
-        CommandHistory(std::size_t capacity = 100)
+        CommandHistory(std::size_t capacity = 5000)
         {
             this->capacity = capacity;
         }
@@ -114,6 +115,12 @@ namespace  DFHack
         {
             history.pop_front();
         }
+        /// adds the current list of entries to the given vector
+        void getEntries(std::vector<std::string> &entries)
+        {
+            for (auto &entry : history)
+                entries.push_back(entry);
+        }
     private:
         std::size_t capacity;
         std::deque <std::string> history;

library/include/Core.h

@@ -58,7 +58,6 @@ namespace DFHack
     class Process;
     class Module;
     class Materials;
-    class Notes;
     struct VersionInfo;
     class VersionInfoFactory;
     class PluginManager;
@@ -69,10 +68,6 @@ namespace DFHack
     namespace Lua { namespace Core {
         DFHACK_EXPORT void Reset(color_ostream &out, const char *where);
     } }
-    namespace Windows
-    {
-        class df_window;
-    }
 
     namespace Screen
     {
@@ -126,12 +121,6 @@ namespace DFHack
         friend int  ::SDL_Init(uint32_t flags);
         friend int  ::wgetch(WINDOW * w);
 #endif
-        friend int  ::egg_init(void);
-        friend int  ::egg_shutdown(void);
-        friend int  ::egg_tick(void);
-        friend int  ::egg_prerender(void);
-        friend int  ::egg_sdl_event(SDL::Event* event);
-        friend int  ::egg_curses_event(int orig_return);
     public:
         /// Get the single Core instance or make one.
         static Core& getInstance()
@@ -146,8 +135,6 @@ namespace DFHack
 
         /// get the materials module
         Materials * getMaterials();
-        /// get the notes module
-        Notes * getNotes();
         /// get the graphic module
         Graphic * getGraphic();
         /// sets the current hotkey command
@@ -193,7 +180,6 @@ namespace DFHack
 
         std::unique_ptr<DFHack::Process> p;
         std::shared_ptr<DFHack::VersionInfo> vinfo;
-        DFHack::Windows::df_window * screen_window;
 
         static void print(const char *format, ...) Wformat(printf,1,2);
         static void printerr(const char *format, ...) Wformat(printf,1,2);
@@ -213,7 +199,6 @@ namespace DFHack
 
         bool Init();
         int Update (void);
-        int TileUpdate (void);
         int Shutdown (void);
         int DFH_SDL_Event(SDL::Event* event);
         bool ncurses_wgetch(int in, int & out);
@@ -242,7 +227,6 @@ namespace DFHack
         struct
         {
             Materials * pMaterials;
-            Notes * pNotes;
             Graphic * pGraphic;
         } s_mods;
         std::vector<std::unique_ptr<Module>> allModules;

library/include/DFHack.h

@@ -54,7 +54,6 @@ distribution.
 
 // DFHack modules
 #include "modules/Buildings.h"
-#include "modules/Engravings.h"
 #include "modules/Materials.h"
 #include "modules/Constructions.h"
 #include "modules/Units.h"

library/include/Hooks.h

@@ -47,7 +47,6 @@ namespace SDL
 // these functions are here because they call into DFHack::Core and therefore need to
 // be declared as friend functions/known
 #ifdef _DARWIN
-#include "modules/Graphic.h"
 DFhackCExport int DFH_SDL_NumJoysticks(void);
 DFhackCExport void DFH_SDL_Quit(void);
 DFhackCExport int DFH_SDL_PollEvent(SDL::Event* event);
@@ -75,21 +74,3 @@ DFhackCExport void * SDL_GetVideoSurface(void);
 
 DFhackCExport int SDL_SemWait(vPtr sem);
 DFhackCExport int SDL_SemPost(vPtr sem);
-
-// hook - called early from DF's main()
-DFhackCExport int egg_init(void);
-
-// hook - called before rendering
-DFhackCExport int egg_shutdown(void);
-
-// hook - called for each game tick (or more often)
-DFhackCExport int egg_tick(void);
-
-// hook - called before rendering
-DFhackCExport int egg_prerender(void);
-
-// hook - called for each SDL event, can filter both the event and the return value
-DFhackCExport int egg_sdl_event(SDL::Event* event);
-
-// hook - ncurses event. return -1 to consume
-DFhackCExport int egg_curses_event(int orig_return);

library/include/LuaTools.h

@@ -339,6 +339,8 @@ namespace DFHack {namespace Lua {
         }
     }
 
+    DFHACK_EXPORT void GetVector(lua_State *state, std::vector<std::string> &pvec);
+
     DFHACK_EXPORT int PushPosXYZ(lua_State *state, df::coord pos);
     DFHACK_EXPORT int PushPosXY(lua_State *state, df::coord2d pos);
 

library/include/ModuleFactory.h

@@ -33,7 +33,6 @@ namespace DFHack
 {
     class Module;
     std::unique_ptr<Module> createMaterials();
-    std::unique_ptr<Module> createNotes();
     std::unique_ptr<Module> createGraphic();
 }
 #endif

library/include/modules/Constructions.h

@@ -42,23 +42,7 @@ namespace DFHack
 {
 namespace Constructions
 {
-// "Simplified" copy of construction
-struct t_construction {
-    df::coord pos;
-    df::item_type item_type;
-    int16_t item_subtype;
-    int16_t mat_type;
-    int32_t mat_index;
-    df::construction_flags flags;
-    int16_t original_tile;
-    // Pointer to original object, in case you want to modify it
-    df::construction *origin;
-};
 
-DFHACK_EXPORT bool isValid();
-DFHACK_EXPORT uint32_t getCount();
-DFHACK_EXPORT bool copyConstruction (const int32_t index, t_construction &out);
-DFHACK_EXPORT df::construction * getConstruction (const int32_t index);
 DFHACK_EXPORT df::construction * findAtTile(df::coord pos);
 
 DFHACK_EXPORT bool designateNew(df::coord pos, df::construction_type type,

library/include/modules/Engravings.h

@@ -1,63 +0,0 @@
-/*
-https://github.com/peterix/dfhack
-Copyright (c) 2009-2012 Petr Mrázek (peterix@gmail.com)
-
-This software is provided 'as-is', without any express or implied
-warranty. In no event will the authors be held liable for any
-damages arising from the use of this software.
-
-Permission is granted to anyone to use this software for any
-purpose, including commercial applications, and to alter it and
-redistribute it freely, subject to the following restrictions:
-
-1. The origin of this software must not be misrepresented; you must
-not claim that you wrote the original software. If you use this
-software in a product, an acknowledgment in the product documentation
-would be appreciated but is not required.
-
-2. Altered source versions must be plainly marked as such, and
-must not be misrepresented as being the original software.
-
-3. This notice may not be removed or altered from any source
-distribution.
-*/
-
-#pragma once
-#ifndef CL_MOD_ENGRAVINGS
-#define CL_MOD_ENGRAVINGS
-/*
-* DF engravings
-*/
-#include "Export.h"
-#include "DataDefs.h"
-#include "df/engraving.h"
-/**
- * \defgroup grp_engraving Engraving module parts
- * @ingroup grp_modules
- */
-namespace DFHack
-{
-namespace Engravings
-{
-// "Simplified" copy of engraving
-struct t_engraving {
-    int32_t artist;
-    int32_t masterpiece_event;
-    int32_t skill_rating;
-    df::coord pos;
-    df::engraving_flags flags;
-    int8_t tile;
-    int32_t art_id;
-    int16_t art_subid;
-    df::item_quality quality;
-    // Pointer to original object, in case you want to modify it
-    df::engraving *origin;
-};
-
-DFHACK_EXPORT bool isValid();
-DFHACK_EXPORT uint32_t getCount();
-DFHACK_EXPORT bool copyEngraving (const int32_t index, t_engraving &out);
-DFHACK_EXPORT df::engraving * getEngraving (const int32_t index);
-}
-}
-#endif

library/include/modules/Notes.h

@@ -1,65 +0,0 @@
-#pragma once
-#ifndef CL_MOD_NOTES
-#define CL_MOD_NOTES
-/**
- * \defgroup grp_notes In game notes (and routes)
- * @ingroup grp_notes
- */
-#include "Export.h"
-#include "Module.h"
-
-#include <vector>
-#include <string>
-
-#ifdef __cplusplus
-namespace DFHack
-{
-#endif
-    /**
-     * Game's structure for a note.
-     * \ingroup grp_notes
-     */
-    struct t_note
-    {
-        // First note created has id 0, second has id 1, etc.  Not affected
-        // by lower id notes being deleted.
-        uint32_t id; // 0
-        uint8_t  symbol; // 4
-        uint8_t  unk1; // alignment padding?
-        uint16_t foreground; // 6
-        uint16_t background; // 8
-        uint16_t unk2; // alignment padding?
-
-        std::string name; // C
-        std::string text; // 10
-
-        uint16_t x; // 14
-        uint16_t y; // 16
-        uint16_t z; // 18
-
-        // Is there more?
-    };
-
-#ifdef __cplusplus
-
-    /**
-     * The notes module - allows reading DF in-game notes
-     * \ingroup grp_modules
-     * \ingroup grp_notes
-     */
-    class DFHACK_EXPORT Notes : public Module
-    {
-        public:
-        Notes();
-        ~Notes(){};
-        bool Finish()
-        {
-            return true;
-        }
-        std::vector<t_note*>* notes;
-    };
-
-}
-#endif // __cplusplus
-
-#endif

library/include/modules/Windows.h

@@ -1,270 +0,0 @@
-/*
-https://github.com/peterix/dfhack
-Copyright (c) 2009-2012 Petr Mrázek (peterix@gmail.com)
-
-This software is provided 'as-is', without any express or implied
-warranty. In no event will the authors be held liable for any
-damages arising from the use of this software.
-
-Permission is granted to anyone to use this software for any
-purpose, including commercial applications, and to alter it and
-redistribute it freely, subject to the following restrictions:
-
-1. The origin of this software must not be misrepresented; you must
-not claim that you wrote the original software. If you use this
-software in a product, an acknowledgment in the product documentation
-would be appreciated but is not required.
-
-2. Altered source versions must be plainly marked as such, and
-must not be misrepresented as being the original software.
-
-3. This notice may not be removed or altered from any source
-distribution.
-*/
-
-#pragma once
-#include "Export.h"
-#include <cstddef>
-
-namespace DFHack
-{
-namespace Windows
-{
-    /*
-     * DF window stuffs
-     */
-    enum df_color
-    {
-        black,
-        blue,
-        green,
-        cyan,
-        red,
-        magenta,
-        brown,
-        lgray,
-        dgray,
-        lblue,
-        lgreen,
-        lcyan,
-        lred,
-        lmagenta,
-        yellow,
-        white
-        // maybe add transparency?
-    };
-
-    // The tile format DF uses internally
-    struct df_screentile
-    {
-        uint8_t symbol;
-        uint8_t foreground; ///< df_color
-        uint8_t background; ///< df_color
-        uint8_t bright;
-    };
-
-
-    // our silly painter things and window things follow.
-    class df_window;
-    struct df_tilebuf
-    {
-        df_screentile * data;
-        unsigned int width;
-        unsigned int height;
-    };
-
-    DFHACK_EXPORT df_screentile *getScreenBuffer();
-
-    class DFHACK_EXPORT painter
-    {
-        friend class df_window;
-    public:
-        df_screentile* get(unsigned int x, unsigned int y)
-        {
-            if(x >= width || y >= height)
-                return 0;
-            return &buffer[x*height + y];
-        };
-        bool set(unsigned int x, unsigned int y, df_screentile tile )
-        {
-            if(x >= width || y >= height)
-                return false;
-            buffer[x*height + y] = tile;
-            return true;
-        }
-        df_color foreground (df_color change = (df_color) -1)
-        {
-            if(change != -1)
-                current_foreground = change;
-            return current_foreground;
-        }
-        df_color background (df_color change = (df_color) -1)
-        {
-            if(change != -1)
-                current_background = change;
-            return current_background;
-        }
-        void bright (bool change)
-        {
-            current_bright = change;
-        }
-        bool bright ()
-        {
-            return current_bright;
-        }
-        void printStr(std::string & str, bool wrap = false)
-        {
-            for ( auto iter = str.begin(); iter != str.end(); iter++)
-            {
-                auto elem = *iter;
-                if(cursor_y >= (int)height)
-                    break;
-                if(wrap)
-                {
-                    if(cursor_x >= (int)width)
-                        cursor_x = wrap_column;
-                }
-                df_screentile & tile = buffer[cursor_x * height + cursor_y];
-                tile.symbol = elem;
-                tile.foreground = current_foreground;
-                tile.background = current_background;
-                tile.bright = current_bright;
-                cursor_x++;
-            }
-        }
-        void set_wrap (int new_column)
-        {
-            wrap_column = new_column;
-        }
-        void gotoxy(unsigned int x, unsigned int y)
-        {
-            cursor_x = x;
-            cursor_y = y;
-        }
-        void reset()
-        {
-            cursor_x = 0;
-            cursor_y = 0;
-            current_background = black;
-            current_foreground = white;
-            current_bright = false;
-            wrap_column = 0;
-        }
-    private:
-        painter (df_window * orig, df_screentile * buf, unsigned int width, unsigned int height)
-        {
-            origin = orig;
-            this->width = width;
-            this->height = height;
-            this->buffer = buf;
-            reset();
-        }
-        df_window* origin;
-        unsigned int width;
-        unsigned int height;
-        df_screentile* buffer;
-        // current paint cursor position
-        int cursor_x;
-        int cursor_y;
-        int wrap_column;
-        // current foreground color
-        df_color current_foreground;
-        // current background color
-        df_color current_background;
-        // make bright?
-        bool current_bright;
-    };
-
-    class DFHACK_EXPORT df_window
-    {
-        friend class painter;
-    public:
-        df_window(int x, int y, unsigned int width, unsigned int height);
-        virtual ~df_window();
-        virtual bool move (int left_, int top_, unsigned int width_, unsigned int height_) = 0;
-        virtual void paint () = 0;
-        virtual painter * lock();
-        bool unlock (painter * painter);
-        virtual bool addChild(df_window *);
-        virtual df_tilebuf getBuffer() = 0;
-    public:
-        df_screentile* buffer;
-        unsigned int width;
-        unsigned int height;
-    protected:
-        df_window * parent;
-        std::vector <df_window *> children;
-        int left;
-        int top;
-        // FIXME: FAKE
-        bool locked;
-        painter * current_painter;
-    };
-
-    class DFHACK_EXPORT top_level_window : public df_window
-    {
-    public:
-        top_level_window();
-        virtual bool move (int left_, int top_, unsigned int width_, unsigned int height_);
-        virtual void paint ();
-        virtual painter * lock();
-        virtual df_tilebuf getBuffer();
-    };
-    class DFHACK_EXPORT buffered_window : public df_window
-    {
-    public:
-        buffered_window(int x, int y, unsigned int width, unsigned int height):df_window(x,y,width, height)
-        {
-            buffer = new df_screentile[width*height];
-        };
-        virtual ~buffered_window()
-        {
-            delete buffer;
-        }
-        virtual void blit_to_parent ()
-        {
-            df_tilebuf par = parent->getBuffer();
-            for(unsigned xi = 0; xi < width; xi++)
-            {
-                for(unsigned yi = 0; yi < height; yi++)
-                {
-                    unsigned parx = left + xi;
-                    unsigned pary = top + yi;
-                    if(pary >= par.height) continue;
-                    if(parx >= par.width) continue;
-                    par.data[parx * par.height + pary] = buffer[xi * height + yi];
-                }
-            }
-        }
-        virtual df_tilebuf getBuffer()
-        {
-            df_tilebuf buf;
-            buf.data = buffer;
-            buf.width = width;
-            buf.height = height;
-            return buf;
-        };
-    };
-    class DFHACK_EXPORT dfhack_dummy : public buffered_window
-    {
-    public:
-        dfhack_dummy(int x, int y):buffered_window(x,y,6,1){};
-        virtual bool move (int left_, int top_, unsigned int width_, unsigned int height_)
-        {
-            top = top_;
-            left = left_;
-            return true;
-        }
-        virtual void paint ()
-        {
-            painter * p = lock();
-            p->bright(true);
-            p->background(black);
-            p->foreground(white);
-            std::string dfhack = "DFHack";
-            p->printStr(dfhack);
-            blit_to_parent();
-        }
-    };
-}
-}

library/lua/argparse.lua

@@ -154,11 +154,20 @@ function numberList(arg, arg_name, list_length)
     return strings
 end
 
--- throws if val is not a nonnegative integer; otherwise returns val
+function positiveInt(arg, arg_name)
-local function check_nonnegative_int(val, arg_name)
+    local val = tonumber(arg)
+    if not val or val <= 0 or val ~= math.floor(val) then
+        arg_error(arg_name,
+                  'expected positive integer; got "%s"', tostring(arg))
+    end
+    return val
+end
+
+function nonnegativeInt(arg, arg_name)
+    local val = tonumber(arg)
     if not val or val < 0 or val ~= math.floor(val) then
         arg_error(arg_name,
-                  'expected non-negative integer; got "%s"', tostring(val))
+                  'expected non-negative integer; got "%s"', tostring(arg))
     end
     return val
 end
@@ -177,9 +186,9 @@ function coords(arg, arg_name, skip_validation)
         return cursor
     end
     local numbers = numberList(arg, arg_name, 3)
-    local pos = xyz2pos(check_nonnegative_int(numbers[1]),
+    local pos = xyz2pos(nonnegativeInt(numbers[1]),
-                        check_nonnegative_int(numbers[2]),
+                        nonnegativeInt(numbers[2]),
-                        check_nonnegative_int(numbers[3]))
+                        nonnegativeInt(numbers[3]))
     if not skip_validation and not dfhack.maps.isValidTilePos(pos) then
         arg_error(arg_name,
                   'specified coordinates not on current map: "%s"', arg)

library/lua/gui/materials.lua

@@ -345,7 +345,8 @@ end
 
 function ItemTraitsDialog(args)
     local job_item_flags_map = {}
-    for i = 1, 3 do
+    for i = 1, 5 do
+        if not df['job_item_flags'..i] then break end
         for _, f in ipairs(df['job_item_flags'..i]) do
             if f then
                 job_item_flags_map[f] = 'flags'..i
@@ -600,6 +601,99 @@ function ItemTraitsDialog(args)
         args.on_select = function(idx, obj)
             return cb(obj)
         end
+    else
+        local function toggleFlag(obj, ffield, flag)
+            local job_item = obj.job_item
+            job_item[ffield][flag] = not job_item[ffield][flag]
+        end
+
+        local function toggleToolUse(obj, tool_use)
+            local job_item = obj.job_item
+            tool_use = df.tool_uses[tool_use]
+            if job_item.has_tool_use == tool_use then
+                job_item.has_tool_use = df.tool_uses.NONE
+            else
+                job_item.has_tool_use = tool_use
+            end
+        end
+
+        local function toggleMetalOre(obj, ore_ix)
+            local job_item = obj.job_item
+            if job_item.metal_ore == ore_ix then
+                job_item.metal_ore = -1
+            else
+                job_item.metal_ore = ore_ix
+            end
+        end
+
+        function toggleReactionClass(obj, reaction_class)
+            local job_item = obj.job_item
+            if job_item.reaction_class == reaction_class then
+                job_item.reaction_class = ''
+            else
+                job_item.reaction_class = reaction_class
+            end
+        end
+
+        local function toggleProductMaterial(obj, product_materials)
+            local job_item = obj.job_item
+            if job_item.has_material_reaction_product == product_materials then
+                job_item.has_material_reaction_product = ''
+            else
+                job_item.has_material_reaction_product = product_materials
+            end
+        end
+
+        local function unsetFlags(obj)
+            local job_item = obj.job_item
+            for flag, ffield in pairs(job_item_flags_map) do
+                if job_item[ffield][flag] then
+                    toggleFlag(obj, ffield, flag)
+                end
+            end
+        end
+
+        local function setTrait(obj, sel)
+            if sel.ffield then
+                --print('toggle flag', sel.ffield, sel.flag)
+                toggleFlag(obj, sel.ffield, sel.flag)
+            elseif sel.reset_flags then
+                --print('reset every flag')
+                unsetFlags(obj)
+            elseif sel.tool_use then
+                --print('toggle tool_use', sel.tool_use)
+                toggleToolUse(obj, sel.tool_use)
+            elseif sel.ore_ix then
+                --print('toggle ore', sel.ore_ix)
+                toggleMetalOre(obj, sel.ore_ix)
+            elseif sel.reaction_class then
+                --print('toggle reaction class', sel.reaction_class)
+                toggleReactionClass(obj, sel.reaction_class)
+            elseif sel.product_materials then
+                --print('toggle product materials', sel.product_materials)
+                toggleProductMaterial(obj, sel.product_materials)
+            elseif sel.reset_all_traits then
+                --print('reset every trait')
+                -- flags
+                unsetFlags(obj)
+                -- tool use
+                toggleToolUse(obj, 'NONE')
+                -- metal ore
+                toggleMetalOre(obj, -1)
+                -- reaction class
+                toggleReactionClass(obj, '')
+                -- producing
+                toggleProductMaterial(obj, '')
+            else
+                print('unknown sel')
+                printall(sel)
+                error('Selected entry in ItemTraitsDialog was of unknown type')
+            end
+        end
+
+        args.on_select = function(idx, choice)
+            setTrait(args, choice)
+        end
     end
 
     return dlg.ListBox(args)

library/lua/gui/widgets.lua

@@ -121,12 +121,12 @@ ResizingPanel = defclass(ResizingPanel, Panel)
 -- adjust our frame dimensions according to positions and sizes of our subviews
 function ResizingPanel:postUpdateLayout(frame_body)
     local w, h = 0, 0
-    for _,subview in ipairs(self.subviews) do
+    for _,s in ipairs(self.subviews) do
-        if subview.visible then
+        if s.visible then
-            w = math.max(w, (subview.frame.l or 0) +
+            w = math.max(w, (s.frame and s.frame.l or 0) +
-                            (subview.frame.w or frame_body.width))
+                            (s.frame and s.frame.w or frame_body.width))
-            h = math.max(h, (subview.frame.t or 0) +
+            h = math.max(h, (s.frame and s.frame.t or 0) +
-                            (subview.frame.h or frame_body.height))
+                            (s.frame and s.frame.h or frame_body.height))
         end
     end
     if not self.frame then self.frame = {} end
@@ -184,18 +184,26 @@ EditField.ATTRS{
     on_char = DEFAULT_NIL,
     on_change = DEFAULT_NIL,
     on_submit = DEFAULT_NIL,
+    on_submit2 = DEFAULT_NIL,
     key = DEFAULT_NIL,
     key_sep = DEFAULT_NIL,
-    frame = {h=1},
     modal = false,
 }
 
+function EditField:preinit(init_table)
+    init_table.frame = init_table.frame or {}
+    init_table.frame.h = init_table.frame.h or 1
+end
+
 function EditField:init()
     local function on_activate()
         self.saved_text = self.text
         self:setFocus(true)
     end
 
+    self.start_pos = 1
+    self.cursor = #self.text + 1
+
     self:addviews{HotkeyLabel{frame={t=0,l=0},
                               key=self.key,
                               key_sep=self.key_sep,
@@ -207,6 +215,19 @@ function EditField:getPreferredFocusState()
     return not self.key
 end
 
+function EditField:setCursor(cursor)
+    if not cursor or cursor > #self.text then
+        self.cursor = #self.text + 1
+        return
+    end
+    self.cursor = math.max(1, cursor)
+end
+
+function EditField:setText(text, cursor)
+    self.text = text
+    self:setCursor(cursor)
+end
+
 function EditField:postUpdateLayout()
     self.text_offset = self.subviews[1]:getTextWidth()
 end
@@ -214,14 +235,31 @@ end
 function EditField:onRenderBody(dc)
     dc:pen(self.text_pen or COLOR_LIGHTCYAN):fill(0,0,dc.width-1,0)
 
-    local cursor = '_'
+    local cursor_char = '_'
     if not self.active or not self.focus or gui.blink_visible(300) then
-        cursor = ' '
+        cursor_char = (self.cursor > #self.text) and ' ' or
+                                        self.text:sub(self.cursor, self.cursor)
     end
-    local txt = self.text .. cursor
+    local txt = self.text:sub(1, self.cursor - 1) .. cursor_char ..
+                                                self.text:sub(self.cursor + 1)
     local max_width = dc.width - self.text_offset
+    self.start_pos = 1
     if #txt > max_width then
-        txt = string.char(27)..string.sub(txt, #txt-max_width+2)
+        -- get the substring in the vicinity of the cursor
+        max_width = max_width - 2
+        local half_width = math.floor(max_width/2)
+        local start_pos = math.max(1, self.cursor-half_width)
+        local end_pos = math.min(#txt, self.cursor+half_width-1)
+        if self.cursor + half_width > #txt then
+            start_pos = #txt - (max_width - 1)
+        end
+        if self.cursor - half_width <= 1 then
+            end_pos = max_width + 1
+        end
+        self.start_pos = start_pos > 1 and start_pos - 1 or start_pos
+        txt = ('%s%s%s'):format(start_pos == 1 and '' or string.char(27),
+                                txt:sub(start_pos, end_pos),
+                                end_pos == #txt and '' or string.char(26))
     end
     dc:advance(self.text_offset):string(txt)
 end
@@ -234,7 +272,7 @@ function EditField:onInput(keys)
 
     if self.key and keys.LEAVESCREEN then
         local old = self.text
-        self.text = self.saved_text
+        self:setText(self.saved_text)
         if self.on_change and old ~= self.saved_text then
             self.on_change(self.text, old)
         end
@@ -251,23 +289,62 @@ function EditField:onInput(keys)
             return true
         end
         return not not self.key
+    elseif keys.SEC_SELECT then
+        if self.key then
+            self:setFocus(false)
         end
-
+        if self.on_submit2 then
-    if keys._STRING then
+            self.on_submit2(self.text)
+            return true
+        end
+        return not not self.key
+    elseif keys._MOUSE_L then
+        local mouse_x, mouse_y = self:getMousePos()
+        if mouse_x then
+            self:setCursor(self.start_pos + mouse_x)
+            return true
+        end
+    elseif keys._STRING then
         local old = self.text
         if keys._STRING == 0 then
             -- handle backspace
-            self.text = string.sub(old, 1, #old-1)
+            local del_pos = self.cursor - 1
+            if del_pos > 0 then
+                self:setText(old:sub(1, del_pos-1) .. old:sub(del_pos+1),
+                             del_pos)
+            end
         else
             local cv = string.char(keys._STRING)
             if not self.on_char or self.on_char(cv, old) then
-                self.text = old .. cv
+                self:setText(old:sub(1,self.cursor-1)..cv..old:sub(self.cursor),
+                             self.cursor + 1)
             end
         end
         if self.on_change and self.text ~= old then
             self.on_change(self.text, old)
         end
         return true
+    elseif keys.CURSOR_LEFT then
+        self:setCursor(self.cursor - 1)
+        return true
+    elseif keys.A_MOVE_W_DOWN then -- Ctrl-Left (end of prev word)
+        local _, prev_word_end = self.text:sub(1, self.cursor-1):
+                                                    find('.*[%w_%-][^%w_%-]')
+        self:setCursor(prev_word_end or 1)
+        return true
+    elseif keys.A_CARE_MOVE_W then -- Alt-Left (home)
+        self:setCursor(1)
+        return true
+    elseif keys.CURSOR_RIGHT then
+        self:setCursor(self.cursor + 1)
+        return true
+    elseif keys.A_MOVE_E_DOWN then -- Ctrl-Right (beginning of next word)
+        local _,next_word_start = self.text:find('[^%w_%-][%w_%-]', self.cursor)
+        self:setCursor(next_word_start)
+        return true
+    elseif keys.A_CARE_MOVE_E then -- Alt-Right (end)
+        self:setCursor()
+        return true
     end
 
     -- if we're modal, then unconditionally eat all the input
@@ -469,7 +546,6 @@ Label.ATTRS{
 }
 
 function Label:init(args)
-    self.start_line_num = 1
     -- use existing saved text if no explicit text was specified. this avoids
     -- overwriting pre-formatted text that subclasses may have already set
     self:setText(args.text or self.text)
@@ -479,6 +555,7 @@ function Label:init(args)
 end
 
 function Label:setText(text)
+    self.start_line_num = 1
     self.text = text
     parse_label_text(self)
 
@@ -575,6 +652,19 @@ function Label:onRenderFrame(dc, rect)
 end
 
 function Label:scroll(nlines)
+    if type(nlines) == 'string' then
+        if nlines == '+page' then
+            nlines = self.frame_body.height
+        elseif nlines == '-page' then
+            nlines = -self.frame_body.height
+        elseif nlines == '+halfpage' then
+            nlines = math.ceil(self.frame_body.height/2)
+        elseif nlines == '-halfpage' then
+            nlines = -math.ceil(self.frame_body.height/2)
+        else
+            error(('unhandled scroll keyword: "%s"'):format(nlines))
+        end
+    end
     local n = self.start_line_num + nlines
     n = math.min(n, self:getTextHeight() - self.frame_body.height + 1)
     n = math.max(n, 1)
@@ -591,11 +681,6 @@ function Label:onInput(keys)
     end
     for k,v in pairs(self.scroll_keys) do
         if keys[k] then
-            if v == '+page' then
-                v = self.frame_body.height
-            elseif v == '-page' then
-                v = -self.frame_body.height
-            end
             self:scroll(v)
         end
     end
@@ -671,6 +756,15 @@ function HotkeyLabel:init()
                   on_activate=self.on_activate}}
 end
 
+function HotkeyLabel:onInput(keys)
+    if HotkeyLabel.super.onInput(self, keys) then
+        return true
+    elseif keys._MOUSE_L and self:getMousePos() then
+        self.on_activate()
+        return true
+    end
+end
+
 ----------------------
 -- CycleHotkeyLabel --
 ----------------------
@@ -743,6 +837,15 @@ function CycleHotkeyLabel:getOptionValue(option_idx)
     return option
 end
 
+function CycleHotkeyLabel:onInput(keys)
+    if CycleHotkeyLabel.super.onInput(self, keys) then
+        return true
+    elseif keys._MOUSE_L and self:getMousePos() then
+        self:cycle()
+        return true
+    end
+end
+
 -----------------------
 -- ToggleHotkeyLabel --
 -----------------------
@@ -949,6 +1052,15 @@ function List:onInput(keys)
     elseif self.on_submit2 and keys.SEC_SELECT then
         self:submit2()
         return true
+    elseif keys._MOUSE_L then
+        local _, mouse_y = self:getMousePos()
+        if mouse_y and #self.choices > 0 and
+                mouse_y < (#self.choices-self.page_top+1) * self.row_height then
+            local idx = self.page_top + math.floor(mouse_y/self.row_height)
+            self:setSelected(idx)
+            self:submit()
+            return true
+        end
     else
         for k,v in pairs(self.scroll_keys) do
             if keys[k] then
@@ -1050,7 +1162,7 @@ end
 
 function FilteredList:setChoices(choices, pos)
     choices = choices or {}
-    self.edit.text = ''
+    self.edit:setText('')
     self.list:setChoices(choices, pos)
     self.choices = self.list.choices
     self.not_found.visible = (#choices == 0)
@@ -1092,7 +1204,7 @@ function FilteredList:setFilter(filter, pos)
     local cidx = nil
 
     filter = filter or ''
-    self.edit.text = filter
+    self.edit:setText(filter)
 
     if filter ~= '' then
         local tokens = filter:split()

library/lua/helpdb.lua

@@ -0,0 +1,735 @@
+-- The help text database and query interface.
+--
+-- Help text is read from the rendered text in hack/docs/docs/. If no rendered
+-- text exists, it is read from the script sources (for scripts) or the string
+-- passed to the PluginCommand initializer (for plugins).
+--
+-- There should be one help file for each plugin that contains a summary for the
+-- plugin itself and help for all the commands that plugin provides (if any).
+-- Each script should also have one documentation file.
+--
+-- The database is lazy-loaded when an API method is called. It rechecks its
+-- help sources for updates if an API method has not been called in the last
+-- 60 seconds.
+
+local _ENV = mkmodule('helpdb')
+
+local MAX_STALE_MS = 60000
+
+-- paths
+local RENDERED_PATH = 'hack/docs/docs/tools/'
+local TAG_DEFINITIONS = 'hack/docs/docs/Tags.txt'
+
+-- used when reading help text embedded in script sources
+local SCRIPT_DOC_BEGIN = '[====['
+local SCRIPT_DOC_END = ']====]'
+local SCRIPT_DOC_BEGIN_RUBY = '=begin'
+local SCRIPT_DOC_END_RUBY = '=end'
+
+-- enums
+local ENTRY_TYPES = {
+    BUILTIN='builtin',
+    PLUGIN='plugin',
+    COMMAND='command'
+}
+
+local HELP_SOURCES = {
+    RENDERED='rendered', -- from the installed, rendered help text
+    PLUGIN='plugin',     -- from the plugin source code
+    SCRIPT='script',     -- from the script source code
+    STUB='stub',         -- from a generated stub
+}
+
+-- builtin command names, with aliases mapped to their canonical form
+local BUILTINS = {
+    ['?']='help',
+    alias=true,
+    clear='cls',
+    cls=true,
+    ['devel/dump-rpc']=true,
+    die=true,
+    dir='ls',
+    disable=true,
+    enable=true,
+    fpause=true,
+    help=true,
+    hide=true,
+    keybinding=true,
+    ['kill-lua']=true,
+    ['load']=true,
+    ls=true,
+    man='help',
+    plug=true,
+    reload=true,
+    script=true,
+    ['sc-script']=true,
+    show=true,
+    tags=true,
+    ['type']=true,
+    unload=true,
+}
+
+---------------------------------------------------------------------------
+-- data structures
+---------------------------------------------------------------------------
+
+-- help text database, keys are a subset of the entry database
+-- entry name -> {
+--   help_source (element of HELP_SOURCES),
+--   short_help (string),
+--   long_help (string),
+--   tags (set),
+--   source_timestamp (mtime, 0 for non-files),
+--   source_path (string, nil for non-files)
+-- }
+local textdb = {}
+
+-- entry database, points to text in textdb
+-- entry name -> {
+--   entry_types (set of ENTRY_TYPES),
+--   short_help (string, if not nil then overrides short_help in text_entry),
+--   text_entry (string)
+-- }
+--
+-- entry_types is a set because plugin commands can also be the plugin names.
+local entrydb = {}
+
+
+-- tag name -> list of entry names
+-- Tags defined in the TAG_DEFINITIONS file that have no associated db entries
+-- will have an empty list.
+local tag_index = {}
+
+---------------------------------------------------------------------------
+-- data ingestion
+---------------------------------------------------------------------------
+
+local function get_rendered_path(entry_name)
+    return RENDERED_PATH .. entry_name .. '.txt'
+end
+
+local function has_rendered_help(entry_name)
+    return dfhack.filesystem.mtime(get_rendered_path(entry_name)) ~= -1
+end
+
+local DEFAULT_HELP_TEMPLATE = [[
+%s
+%s
+
+No help available.
+]]
+local function make_default_entry(entry_name, help_source, kwargs)
+    local default_long_help = DEFAULT_HELP_TEMPLATE:format(
+                                            entry_name, ('*'):rep(#entry_name))
+    return {
+        help_source=help_source,
+        short_help='No help available.',
+        long_help=default_long_help,
+        tags={},
+        source_timestamp=kwargs.source_timestamp or 0,
+        source_path=kwargs.source_path}
+end
+
+-- updates the short_text, the long_text, and the tags in the given entry based
+-- on the text returned from the iterator.
+-- if defined, opts can have the following fields:
+--   begin_marker (string that marks the beginning of the help text; all text
+--                 before this marker is ignored)
+--   end_marker (string that marks the end of the help text; text will stop
+--               being parsed after this marker is seen)
+--   no_header (don't try to find the entity name at the top of the help text)
+--   first_line_is_short_help (if set, then read the short help text from the
+--                             first commented line of the script instead of
+--                             using the first sentence of the long help text.
+--                             value is the comment character.)
+local function update_entry(entry, iterator, opts)
+    opts = opts or {}
+    local lines, tags = {}, ''
+    local first_line_is_short_help = opts.first_line_is_short_help
+    local begin_marker_found,header_found = not opts.begin_marker,opts.no_header
+    local tags_found, short_help_found = false, opts.skip_short_help
+    local in_tags, in_short_help = false, false
+    for line in iterator do
+        if not short_help_found and first_line_is_short_help then
+            line = line:trim()
+            local _,_,text = line:find('^'..first_line_is_short_help..'%s*(.*)')
+            if not text then
+                -- if no first-line short help found, fall back to getting the
+                -- first sentence of the help text.
+                first_line_is_short_help = false
+            else
+                if not text:endswith('.') then
+                    text = text .. '.'
+                end
+                entry.short_help = text
+                short_help_found = true
+                goto continue
+            end
+        end
+        if not begin_marker_found then
+            local _, endpos = line:find(opts.begin_marker, 1, true)
+            if endpos == #line then
+                begin_marker_found = true
+            end
+            goto continue
+        end
+        if opts.end_marker then
+            local _, endpos = line:find(opts.end_marker, 1, true)
+            if endpos == #line then
+                break
+            end
+        end
+        if not header_found and line:find('%w') then
+            header_found = true
+        elseif in_tags then
+            if #line == 0 then
+                in_tags = false
+            else
+                tags = tags .. line
+            end
+        elseif not tags_found and line:find('^[*]*Tags:[*]*') then
+            _,_,tags = line:trim():find('[*]*Tags:[*]* *(.*)')
+            in_tags, tags_found = true, true
+        elseif not short_help_found and
+                line:find('^%w') then
+            if in_short_help then
+                entry.short_help = entry.short_help .. ' ' .. line
+            else
+                entry.short_help = line
+            end
+            local sentence_end = entry.short_help:find('.', 1, true)
+            if sentence_end then
+                entry.short_help = entry.short_help:sub(1, sentence_end)
+                short_help_found = true
+            else
+                in_short_help = true
+            end
+        end
+        table.insert(lines, line)
+        ::continue::
+    end
+    entry.tags = {}
+    for _,tag in ipairs(tags:split('[ ,|]+')) do
+        if #tag > 0 and tag_index[tag] then
+            entry.tags[tag] = true
+        end
+    end
+    if #lines > 0 then
+        entry.long_help = table.concat(lines, '\n')
+    end
+end
+
+-- create db entry based on parsing sphinx-rendered help text
+local function make_rendered_entry(old_entry, entry_name, kwargs)
+    local source_path = get_rendered_path(entry_name)
+    local source_timestamp = dfhack.filesystem.mtime(source_path)
+    if old_entry and old_entry.help_source == HELP_SOURCES.RENDERED and
+            old_entry.source_timestamp >= source_timestamp then
+        -- we already have the latest info
+        return old_entry
+    end
+    kwargs.source_path, kwargs.source_timestamp = source_path, source_timestamp
+    local entry = make_default_entry(entry_name, HELP_SOURCES.RENDERED, kwargs)
+    local ok, lines = pcall(io.lines, source_path)
+    if not ok then
+        return entry
+    end
+    update_entry(entry, lines)
+    return entry
+end
+
+-- create db entry based on the help text in the plugin source (used by
+-- out-of-tree plugins)
+local function make_plugin_entry(old_entry, entry_name, kwargs)
+    if old_entry and old_entry.source == HELP_SOURCES.PLUGIN then
+        -- we can't tell when a plugin is reloaded, so we can either choose to
+        -- always refresh or never refresh. let's go with never for now for
+        -- performance.
+        return old_entry
+    end
+    local entry = make_default_entry(entry_name, HELP_SOURCES.PLUGIN, kwargs)
+    local long_help = dfhack.internal.getCommandHelp(entry_name)
+    if long_help and #long_help:trim() > 0 then
+        update_entry(entry, long_help:trim():gmatch('[^\n]*'), {no_header=true})
+    end
+    return entry
+end
+
+-- create db entry based on the help text in the script source (used by
+-- out-of-tree scripts)
+local function make_script_entry(old_entry, entry_name, kwargs)
+    local source_path = kwargs.source_path
+    local source_timestamp = dfhack.filesystem.mtime(source_path)
+    if old_entry and old_entry.source == HELP_SOURCES.SCRIPT and
+            old_entry.source_path == source_path and
+            old_entry.source_timestamp >= source_timestamp then
+        -- we already have the latest info
+        return old_entry
+    end
+    kwargs.source_timestamp, kwargs.entry_type = source_timestamp
+    local entry = make_default_entry(entry_name, HELP_SOURCES.SCRIPT, kwargs)
+    local ok, lines = pcall(io.lines, source_path)
+    if not ok then
+        return entry
+    end
+    local is_rb = source_path:endswith('.rb')
+    update_entry(entry, lines,
+            {begin_marker=(is_rb and SCRIPT_DOC_BEGIN_RUBY or SCRIPT_DOC_BEGIN),
+             end_marker=(is_rb and SCRIPT_DOC_BEGIN_RUBY or SCRIPT_DOC_END),
+             first_line_is_short_help=(is_rb and '#' or '%-%-')})
+    return entry
+end
+
+-- updates the dbs (and associated tag index) with a new entry if the entry_name
+-- doesn't already exist in the dbs.
+local function update_db(old_db, entry_name, text_entry, help_source, kwargs)
+    if entrydb[entry_name] then
+        -- already in db (e.g. from a higher-priority script dir); skip
+        return
+    end
+    entrydb[entry_name] = {
+        entry_types=kwargs.entry_types,
+        short_help=kwargs.short_help,
+        text_entry=text_entry
+    }
+    if entry_name ~= text_entry then
+        return
+    end
+
+    local text_entry, old_entry = nil, old_db[entry_name]
+    if help_source == HELP_SOURCES.RENDERED then
+        text_entry = make_rendered_entry(old_entry, entry_name, kwargs)
+    elseif help_source == HELP_SOURCES.PLUGIN then
+        text_entry = make_plugin_entry(old_entry, entry_name, kwargs)
+    elseif help_source == HELP_SOURCES.SCRIPT then
+        text_entry = make_script_entry(old_entry, entry_name, kwargs)
+    elseif help_source == HELP_SOURCES.STUB then
+        text_entry = make_default_entry(entry_name, HELP_SOURCES.STUB, kwargs)
+    else
+        error('unhandled help source: ' .. help_source)
+    end
+    textdb[entry_name] = text_entry
+end
+
+-- add the builtin commands to the db
+local function scan_builtins(old_db)
+    local entry_types = {[ENTRY_TYPES.BUILTIN]=true, [ENTRY_TYPES.COMMAND]=true}
+    for builtin,canonical in pairs(BUILTINS) do
+        if canonical == true then canonical = builtin end
+        update_db(old_db, builtin, canonical,
+            has_rendered_help(canonical) and
+                HELP_SOURCES.RENDERED or HELP_SOURCES.STUB,
+            {entry_types=entry_types})
+    end
+end
+
+-- scan for enableable plugins and plugin-provided commands and add their help
+-- to the db
+local function scan_plugins(old_db)
+    local plugin_names = dfhack.internal.listPlugins()
+    for _,plugin in ipairs(plugin_names) do
+        local commands = dfhack.internal.listCommands(plugin)
+        local includes_plugin = false
+        for _,command in ipairs(commands) do
+            local kwargs = {entry_types={[ENTRY_TYPES.COMMAND]=true}}
+            if command == plugin then
+                kwargs.entry_types[ENTRY_TYPES.PLUGIN]=true
+                includes_plugin = true
+            end
+            kwargs.short_help = dfhack.internal.getCommandDescription(command)
+            update_db(old_db, command, plugin,
+                      has_rendered_help(plugin) and
+                            HELP_SOURCES.RENDERED or HELP_SOURCES.PLUGIN,
+                      kwargs)
+        end
+        if not includes_plugin then
+            update_db(old_db, plugin, plugin,
+                      has_rendered_help(plugin) and
+                            HELP_SOURCES.RENDERED or HELP_SOURCES.STUB,
+                      {entry_types={[ENTRY_TYPES.PLUGIN]=true}})
+        end
+    end
+end
+
+-- scan for scripts and add their help to the db
+local function scan_scripts(old_db)
+    local entry_types = {[ENTRY_TYPES.COMMAND]=true}
+    for _,script_path in ipairs(dfhack.internal.getScriptPaths()) do
+        local files = dfhack.filesystem.listdir_recursive(
+                                            script_path, nil, false)
+        if not files then goto skip_path end
+        for _,f in ipairs(files) do
+            if f.isdir or
+                (not f.path:endswith('.lua') and not f.path:endswith('.rb')) or
+                    f.path:startswith('test/') or
+                    f.path:startswith('internal/') then
+                goto continue
+            end
+            local dot_index = f.path:find('%.[^.]*$')
+            local entry_name = f.path:sub(1, dot_index - 1)
+            local source_path = script_path .. '/' .. f.path
+            update_db(old_db, entry_name, entry_name,
+                      has_rendered_help(entry_name) and
+                            HELP_SOURCES.RENDERED or HELP_SOURCES.SCRIPT,
+                      {entry_types=entry_types, source_path=source_path})
+            ::continue::
+        end
+        ::skip_path::
+    end
+end
+
+-- read tags and descriptions from the TAG_DEFINITIONS file and add them all
+-- to tag_index, initizlizing each entry with an empty list.
+local function initialize_tags()
+    local tag, desc, in_desc = nil, nil, false
+    local ok, lines = pcall(io.lines, TAG_DEFINITIONS)
+    if not ok then return end
+    for line in lines do
+        if in_desc then
+            line = line:trim()
+            if #line == 0 then
+                in_desc = false
+                goto continue
+            end
+            desc = desc .. ' ' .. line
+            tag_index[tag].description = desc
+        else
+            _,_,tag,desc = line:find('^%* (%w+): (.+)')
+            if not tag then goto continue end
+            tag_index[tag] = {description=desc}
+            in_desc = true
+        end
+        ::continue::
+    end
+end
+
+local function index_tags()
+    for entry_name,entry in pairs(entrydb) do
+        for tag in pairs(textdb[entry.text_entry].tags) do
+            -- ignore unknown tags
+            if tag_index[tag] then
+                table.insert(tag_index[tag], entry_name)
+            end
+        end
+    end
+end
+
+-- ensures the db is up to date by scanning all help sources. does not do
+-- anything if it has already been run within the last MAX_STALE_MS milliseconds
+local last_refresh_ms = 0
+local function ensure_db()
+    local now_ms = dfhack.getTickCount()
+    if now_ms - last_refresh_ms <= MAX_STALE_MS then return end
+    last_refresh_ms = now_ms
+
+    local old_db = textdb
+    textdb, entrydb, tag_index = {}, {}, {}
+
+    initialize_tags()
+    scan_builtins(old_db)
+    scan_plugins(old_db)
+    scan_scripts(old_db)
+    index_tags()
+end
+
+---------------------------------------------------------------------------
+-- get API
+---------------------------------------------------------------------------
+
+-- converts strings into single-element lists containing that string
+local function normalize_string_list(l)
+    if not l or #l == 0 then return nil end
+    if type(l) == 'string' then
+        return {l}
+    end
+    return l
+end
+
+local function has_keys(str, dict)
+    if not str or #str == 0 then
+        return false
+    end
+    for _,s in ipairs(normalize_string_list(str)) do
+        if not dict[s] then
+            return false
+        end
+    end
+    return true
+end
+
+-- returns whether the given string (or list of strings) is an entry (are all
+-- entries) in the db
+function is_entry(str)
+    ensure_db()
+    return has_keys(str, entrydb)
+end
+
+local function get_db_property(entry_name, property)
+    ensure_db()
+    if not entrydb[entry_name] then
+        error(('helpdb entry not found: "%s"'):format(entry_name))
+    end
+    return entrydb[entry_name][property] or
+            textdb[entrydb[entry_name].text_entry][property]
+end
+
+function get_entry_types(entry)
+    return get_db_property(entry, 'entry_types')
+end
+
+-- returns the ~54 char summary blurb associated with the entry
+function get_entry_short_help(entry)
+    return get_db_property(entry, 'short_help')
+end
+
+-- returns the full help documentation associated with the entry
+function get_entry_long_help(entry)
+    return get_db_property(entry, 'long_help')
+end
+
+-- returns the set of tags associated with the entry
+function get_entry_tags(entry)
+    return get_db_property(entry, 'tags')
+end
+
+-- returns whether the given string (or list of strings) matches a tag name
+function is_tag(str)
+    ensure_db()
+    return has_keys(str, tag_index)
+end
+
+local function set_to_sorted_list(set)
+    local list = {}
+    for item in pairs(set) do
+        table.insert(list, item)
+    end
+    table.sort(list)
+    return list
+end
+
+-- returns the defined tags in alphabetical order
+function get_tags()
+    ensure_db()
+    return set_to_sorted_list(tag_index)
+end
+
+function get_tag_data(tag)
+    ensure_db()
+    if not tag_index[tag] then
+        error(('helpdb tag not found: "%s"'):format(tag))
+    end
+    return tag_index[tag]
+end
+
+---------------------------------------------------------------------------
+-- search API
+---------------------------------------------------------------------------
+
+-- returns a list of path elements in reverse order
+local function chunk_for_sorting(str)
+    local parts = str:split('/')
+    local chunks = {}
+    for i=1,#parts do
+        chunks[#parts - i + 1] = parts[i]
+    end
+    return chunks
+end
+
+-- sorts by last path component, then by parent path components.
+-- something comes before nothing.
+-- e.g. gui/autofarm comes immediately before autofarm
+function sort_by_basename(a, b)
+    local a = chunk_for_sorting(a)
+    local b = chunk_for_sorting(b)
+    local i = 1
+    while a[i] do
+        if not b[i] then
+            return true
+        end
+        if a[i] ~= b[i] then
+            return a[i] < b[i]
+        end
+        i = i + 1
+    end
+    return false
+end
+
+local function matches(entry_name, filter)
+    if filter.tag then
+        local matched = false
+        local tags = get_db_property(entry_name, 'tags')
+        for _,tag in ipairs(filter.tag) do
+            if tags[tag] then
+                matched = true
+                break
+            end
+        end
+        if not matched then
+            return false
+        end
+    end
+    if filter.entry_type then
+        local matched = false
+        local etypes = get_db_property(entry_name, 'entry_types')
+        for _,etype in ipairs(filter.entry_type) do
+            if etypes[etype] then
+                matched = true
+                break
+            end
+        end
+        if not matched then
+            return false
+        end
+    end
+    if filter.str then
+        local matched = false
+        for _,str in ipairs(filter.str) do
+            if entry_name:find(str, 1, true) then
+                matched = true
+                break
+            end
+        end
+        if not matched then
+            return false
+        end
+    end
+    return true
+end
+
+-- normalizes the lists in the filter and returns nil if no filter elements are
+-- populated
+local function normalize_filter(f)
+    if not f then return nil end
+    local filter = {}
+    filter.str = normalize_string_list(f.str)
+    filter.tag = normalize_string_list(f.tag)
+    filter.entry_type = normalize_string_list(f.entry_type)
+    if not filter.str and not filter.tag and not filter.entry_type then
+        return nil
+    end
+    return filter
+end
+
+-- returns a list of entry names, alphabetized by their last path component,
+-- with populated path components coming before null path components (e.g.
+-- autobutcher will immediately follow gui/autobutcher).
+-- the optional include and exclude filter params are maps with the following
+-- elements:
+--   str - if a string, filters by the given substring. if a table of strings,
+--         includes entry names that match any of the given substrings.
+--   tag - if a string, filters by the given tag name. if a table of strings,
+--         includes entries that match any of the given tags.
+--   entry_type - if a string, matches entries of the given type. if a table of
+--         strings, includes entries that match any of the given types. valid
+--         types are: "builtin", "plugin", "command". note that many plugin
+--         commands have the same name as the plugin, so those entries will
+--         match both "plugin" and "command" types.
+function search_entries(include, exclude)
+    ensure_db()
+    include = normalize_filter(include)
+    exclude = normalize_filter(exclude)
+    local entries = {}
+    for entry in pairs(entrydb) do
+        if (not include or matches(entry, include)) and
+                (not exclude or not matches(entry, exclude)) then
+            table.insert(entries, entry)
+        end
+    end
+    table.sort(entries, sort_by_basename)
+    return entries
+end
+
+-- returns a list of all commands. used by Core's autocomplete functionality.
+function get_commands()
+    local include = {entry_type=ENTRY_TYPES.COMMAND}
+    return search_entries(include)
+end
+
+function is_builtin(command)
+    return is_entry(command) and get_entry_types(command)[ENTRY_TYPES.BUILTIN]
+end
+
+---------------------------------------------------------------------------
+-- print API (outputs to console)
+---------------------------------------------------------------------------
+
+-- implements the 'help' builtin command
+function help(entry)
+    ensure_db()
+    if not entrydb[entry] then
+        dfhack.printerr(('No help entry found for "%s"'):format(entry))
+        return
+    end
+    print(get_entry_long_help(entry))
+end
+
+-- prints col1text (width 21), a one space gap, and col2 (width 58)
+-- if col1text is longer than 21 characters, col2text is printed starting on the
+-- next line. if col2text is longer than 58 characters, it is wrapped. col2text
+-- lines on lines below the col1text output are indented by one space further
+-- than the col2text on the first line.
+local COL1WIDTH, COL2WIDTH = 20, 58
+local function print_columns(col1text, col2text)
+    col2text = col2text:wrap(COL2WIDTH)
+    local wrapped_col2 = {}
+    for line in col2text:gmatch('[^'..NEWLINE..']*') do
+        table.insert(wrapped_col2, line)
+    end
+    local col2_start_line = 1
+    if #col1text > COL1WIDTH then
+        print(col1text)
+    else
+        print(('%-'..COL1WIDTH..'s %s'):format(col1text, wrapped_col2[1]))
+        col2_start_line = 2
+    end
+    for i=col2_start_line,#wrapped_col2 do
+        print(('%'..COL1WIDTH..'s  %s'):format(' ', wrapped_col2[i]))
+    end
+end
+
+-- implements the 'tags' builtin command
+function tags()
+    local tags = get_tags()
+    for _,tag in ipairs(tags) do
+        print_columns(tag, get_tag_data(tag).description)
+    end
+end
+
+-- prints the requested entries to the console. include and exclude filters are
+-- defined as in search_entries() above.
+local function list_entries(skip_tags, include, exclude)
+    local entries = search_entries(include, exclude)
+    for _,entry in ipairs(entries) do
+        print_columns(entry, get_entry_short_help(entry))
+        if not skip_tags then
+            local tags = set_to_sorted_list(get_entry_tags(entry))
+            if #tags > 0 then
+                print(('    tags: %s'):format(table.concat(tags, ', ')))
+            end
+        end
+    end
+    if #entries == 0 then
+        print('No matches.')
+    end
+end
+
+-- wraps the list_entries() API to provide a more convenient interface for Core
+-- to implement the 'ls' builtin command.
+--   filter_str - if a tag name, will filter by that tag. otherwise, will filter
+--                as a substring
+--   skip_tags - whether to skip printing tag info
+--   show_dev_commands - if true, will include scripts in the modtools/ and
+--                       devel/ directories. otherwise those scripts will be
+--                       excluded
+function ls(filter_str, skip_tags, show_dev_commands)
+    local include = {entry_type={ENTRY_TYPES.COMMAND}}
+    if is_tag(filter_str) then
+        include.tag = filter_str
+    else
+        include.str = filter_str
+    end
+    list_entries(skip_tags, include,
+                 show_dev_commands and {} or {tag='dev'})
+end
+
+return _ENV

library/lua/tile-material.lua

@@ -1,25 +1,7 @@
 -- tile-material: Functions to help retrieve the material for a tile.
 
 --[[
-Copyright 2015-2016 Milo Christiansen
+Original code provided by Milo Christiansen in 2015 under the MIT license. Relicensed under the ZLib license to align with the rest of DFHack, with his permission.
-
-This software is provided 'as-is', without any express or implied warranty. In
-no event will the authors be held liable for any damages arising from the use of
-this software.
-
-Permission is granted to anyone to use this software for any purpose, including
-commercial applications, and to alter it and redistribute it freely, subject to
-the following restrictions:
-
-1. The origin of this software must not be misrepresented; you must not claim
-that you wrote the original software. If you use this software in a product, an
-acknowledgment in the product documentation would be appreciated but is not
-required.
-
-2. Altered source versions must be plainly marked as such, and must not be
-misrepresented as being the original software.
-
-3. This notice may not be removed or altered from any source distribution.
 ]]
 
 local _ENV = mkmodule("tile-material")
@@ -85,73 +67,6 @@ local function fixedMat(id)
     end
 end
 
--- BasicMats is a matspec table to pass to GetTileMatSpec or GetTileTypeMat. This particular
--- matspec table covers the common case of returning plant materials for plant tiles and other
--- materials for the remaining tiles.
-BasicMats = {
-    [df.tiletype_material.AIR] = nil, -- Empty
-    [df.tiletype_material.SOIL] = GetLayerMat,
-    [df.tiletype_material.STONE] = GetLayerMat,
-    [df.tiletype_material.FEATURE] = GetFeatureMat,
-    [df.tiletype_material.LAVA_STONE] = GetLavaStone,
-    [df.tiletype_material.MINERAL] = GetVeinMat,
-    [df.tiletype_material.FROZEN_LIQUID] = fixedMat("WATER:NONE"),
-    [df.tiletype_material.CONSTRUCTION] = GetConstructionMat,
-    [df.tiletype_material.GRASS_LIGHT] = GetGrassMat,
-    [df.tiletype_material.GRASS_DARK] = GetGrassMat,
-    [df.tiletype_material.GRASS_DRY] = GetGrassMat,
-    [df.tiletype_material.GRASS_DEAD] = GetGrassMat,
-    [df.tiletype_material.PLANT] = GetShrubMat,
-    [df.tiletype_material.HFS] = nil, -- Eerie Glowing Pit
-    [df.tiletype_material.CAMPFIRE] = GetLayerMat,
-    [df.tiletype_material.FIRE] = GetLayerMat,
-    [df.tiletype_material.ASHES] = GetLayerMat,
-    [df.tiletype_material.MAGMA] = nil, -- SMR
-    [df.tiletype_material.DRIFTWOOD] = GetLayerMat,
-    [df.tiletype_material.POOL] = GetLayerMat,
-    [df.tiletype_material.BROOK] = GetLayerMat,
-    [df.tiletype_material.ROOT] = GetLayerMat,
-    [df.tiletype_material.TREE] = GetTreeMat,
-    [df.tiletype_material.MUSHROOM] = GetTreeMat,
-    [df.tiletype_material.UNDERWORLD_GATE] = nil, -- I guess this is for the gates found in vaults?
-}
-
--- NoPlantMats is a matspec table to pass to GetTileMatSpec or GetTileTypeMat. This particular
--- matspec table will ignore plants, returning layer materials (or nil for trees) instead.
-NoPlantMats = {
-    [df.tiletype_material.SOIL] = GetLayerMat,
-    [df.tiletype_material.STONE] = GetLayerMat,
-    [df.tiletype_material.FEATURE] = GetFeatureMat,
-    [df.tiletype_material.LAVA_STONE] = GetLavaStone,
-    [df.tiletype_material.MINERAL] = GetVeinMat,
-    [df.tiletype_material.FROZEN_LIQUID] = fixedMat("WATER:NONE"),
-    [df.tiletype_material.CONSTRUCTION] = GetConstructionMat,
-    [df.tiletype_material.GRASS_LIGHT] = GetLayerMat,
-    [df.tiletype_material.GRASS_DARK] = GetLayerMat,
-    [df.tiletype_material.GRASS_DRY] = GetLayerMat,
-    [df.tiletype_material.GRASS_DEAD] = GetLayerMat,
-    [df.tiletype_material.PLANT] = GetLayerMat,
-    [df.tiletype_material.CAMPFIRE] = GetLayerMat,
-    [df.tiletype_material.FIRE] = GetLayerMat,
-    [df.tiletype_material.ASHES] = GetLayerMat,
-    [df.tiletype_material.DRIFTWOOD] = GetLayerMat,
-    [df.tiletype_material.POOL] = GetLayerMat,
-    [df.tiletype_material.BROOK] = GetLayerMat,
-    [df.tiletype_material.ROOT] = GetLayerMat,
-}
-
--- OnlyPlantMats is a matspec table to pass to GetTileMatSpec or GetTileTypeMat. This particular
--- matspec table will return nil for any non-plant tile. Plant tiles return the plant material.
-OnlyPlantMats = {
-    [df.tiletype_material.GRASS_LIGHT] = GetGrassMat,
-    [df.tiletype_material.GRASS_DARK] = GetGrassMat,
-    [df.tiletype_material.GRASS_DRY] = GetGrassMat,
-    [df.tiletype_material.GRASS_DEAD] = GetGrassMat,
-    [df.tiletype_material.PLANT] = GetShrubMat,
-    [df.tiletype_material.TREE] = GetTreeMat,
-    [df.tiletype_material.MUSHROOM] = GetTreeMat,
-}
-
 -- GetLayerMat returns the layer material for the given tile.
 -- AFAIK this will never return nil.
 function GetLayerMat(x, y, z)
@@ -349,6 +264,73 @@ function GetFeatureMat(x, y, z)
     return nil
 end
 
+-- BasicMats is a matspec table to pass to GetTileMatSpec or GetTileTypeMat. This particular
+-- matspec table covers the common case of returning plant materials for plant tiles and other
+-- materials for the remaining tiles.
+BasicMats = {
+    [df.tiletype_material.AIR] = nil, -- Empty
+    [df.tiletype_material.SOIL] = GetLayerMat,
+    [df.tiletype_material.STONE] = GetLayerMat,
+    [df.tiletype_material.FEATURE] = GetFeatureMat,
+    [df.tiletype_material.LAVA_STONE] = GetLavaStone,
+    [df.tiletype_material.MINERAL] = GetVeinMat,
+    [df.tiletype_material.FROZEN_LIQUID] = fixedMat("WATER:NONE"),
+    [df.tiletype_material.CONSTRUCTION] = GetConstructionMat,
+    [df.tiletype_material.GRASS_LIGHT] = GetGrassMat,
+    [df.tiletype_material.GRASS_DARK] = GetGrassMat,
+    [df.tiletype_material.GRASS_DRY] = GetGrassMat,
+    [df.tiletype_material.GRASS_DEAD] = GetGrassMat,
+    [df.tiletype_material.PLANT] = GetShrubMat,
+    [df.tiletype_material.HFS] = nil, -- Eerie Glowing Pit
+    [df.tiletype_material.CAMPFIRE] = GetLayerMat,
+    [df.tiletype_material.FIRE] = GetLayerMat,
+    [df.tiletype_material.ASHES] = GetLayerMat,
+    [df.tiletype_material.MAGMA] = nil, -- SMR
+    [df.tiletype_material.DRIFTWOOD] = GetLayerMat,
+    [df.tiletype_material.POOL] = GetLayerMat,
+    [df.tiletype_material.BROOK] = GetLayerMat,
+    [df.tiletype_material.ROOT] = GetLayerMat,
+    [df.tiletype_material.TREE] = GetTreeMat,
+    [df.tiletype_material.MUSHROOM] = GetTreeMat,
+    [df.tiletype_material.UNDERWORLD_GATE] = nil, -- I guess this is for the gates found in vaults?
+}
+
+-- NoPlantMats is a matspec table to pass to GetTileMatSpec or GetTileTypeMat. This particular
+-- matspec table will ignore plants, returning layer materials (or nil for trees) instead.
+NoPlantMats = {
+    [df.tiletype_material.SOIL] = GetLayerMat,
+    [df.tiletype_material.STONE] = GetLayerMat,
+    [df.tiletype_material.FEATURE] = GetFeatureMat,
+    [df.tiletype_material.LAVA_STONE] = GetLavaStone,
+    [df.tiletype_material.MINERAL] = GetVeinMat,
+    [df.tiletype_material.FROZEN_LIQUID] = fixedMat("WATER:NONE"),
+    [df.tiletype_material.CONSTRUCTION] = GetConstructionMat,
+    [df.tiletype_material.GRASS_LIGHT] = GetLayerMat,
+    [df.tiletype_material.GRASS_DARK] = GetLayerMat,
+    [df.tiletype_material.GRASS_DRY] = GetLayerMat,
+    [df.tiletype_material.GRASS_DEAD] = GetLayerMat,
+    [df.tiletype_material.PLANT] = GetLayerMat,
+    [df.tiletype_material.CAMPFIRE] = GetLayerMat,
+    [df.tiletype_material.FIRE] = GetLayerMat,
+    [df.tiletype_material.ASHES] = GetLayerMat,
+    [df.tiletype_material.DRIFTWOOD] = GetLayerMat,
+    [df.tiletype_material.POOL] = GetLayerMat,
+    [df.tiletype_material.BROOK] = GetLayerMat,
+    [df.tiletype_material.ROOT] = GetLayerMat,
+}
+
+-- OnlyPlantMats is a matspec table to pass to GetTileMatSpec or GetTileTypeMat. This particular
+-- matspec table will return nil for any non-plant tile. Plant tiles return the plant material.
+OnlyPlantMats = {
+    [df.tiletype_material.GRASS_LIGHT] = GetGrassMat,
+    [df.tiletype_material.GRASS_DARK] = GetGrassMat,
+    [df.tiletype_material.GRASS_DRY] = GetGrassMat,
+    [df.tiletype_material.GRASS_DEAD] = GetGrassMat,
+    [df.tiletype_material.PLANT] = GetShrubMat,
+    [df.tiletype_material.TREE] = GetTreeMat,
+    [df.tiletype_material.MUSHROOM] = GetTreeMat,
+}
+
 -- GetTileMat will return the material of the specified tile as determined by its tile type and the
 -- world geology data, etc.
 -- The returned material should exactly match the material reported by DF except in cases where is

library/modules/Constructions.cpp

@@ -51,22 +51,6 @@ using namespace DFHack;
 using namespace df::enums;
 using df::global::world;
 
-bool Constructions::isValid()
-{
-    return (world != NULL);
-}
-
-uint32_t Constructions::getCount()
-{
-    return world->constructions.size();
-}
-
-df::construction * Constructions::getConstruction(const int32_t index)
-{
-    if (uint32_t(index) >= getCount())
-        return NULL;
-    return world->constructions[index];
-}
 
 df::construction * Constructions::findAtTile(df::coord pos)
 {
@@ -77,23 +61,6 @@ df::construction * Constructions::findAtTile(df::coord pos)
     return NULL;
 }
 
-bool Constructions::copyConstruction(const int32_t index, t_construction &out)
-{
-    if (uint32_t(index) >= getCount())
-        return false;
-
-    out.origin = world->constructions[index];
-
-    out.pos = out.origin->pos;
-    out.item_type = out.origin->item_type;
-    out.item_subtype = out.origin->item_subtype;
-    out.mat_type = out.origin->mat_type;
-    out.mat_index = out.origin->mat_index;
-    out.flags = out.origin->flags;
-    out.original_tile = out.origin->original_tile;
-    return true;
-}
-
 bool Constructions::designateNew(df::coord pos, df::construction_type type,
                                  df::item_type item, int mat_index)
 {

library/modules/Engravings.cpp

@@ -1,78 +0,0 @@
-/*
-https://github.com/peterix/dfhack
-Copyright (c) 2009-2012 Petr Mrázek (peterix@gmail.com)
-
-This software is provided 'as-is', without any express or implied
-warranty. In no event will the authors be held liable for any
-damages arising from the use of this software.
-
-Permission is granted to anyone to use this software for any
-purpose, including commercial applications, and to alter it and
-redistribute it freely, subject to the following restrictions:
-
-1. The origin of this software must not be misrepresented; you must
-not claim that you wrote the original software. If you use this
-software in a product, an acknowledgment in the product documentation
-would be appreciated but is not required.
-
-2. Altered source versions must be plainly marked as such, and
-must not be misrepresented as being the original software.
-
-3. This notice may not be removed or altered from any source
-distribution.
-*/
-
-
-#include "Internal.h"
-
-#include <string>
-#include <vector>
-#include <map>
-using namespace std;
-
-#include "VersionInfo.h"
-#include "MemAccess.h"
-#include "Types.h"
-#include "Core.h"
-
-#include "modules/Engravings.h"
-#include "df/world.h"
-
-using namespace DFHack;
-using df::global::world;
-
-bool Engravings::isValid()
-{
-    return (world != NULL);
-}
-
-uint32_t Engravings::getCount()
-{
-    return world->engravings.size();
-}
-
-df::engraving * Engravings::getEngraving(int index)
-{
-    if (uint32_t(index) >= getCount())
-        return NULL;
-    return world->engravings[index];
-}
-
-bool Engravings::copyEngraving(const int32_t index, t_engraving &out)
-{
-    if (uint32_t(index) >= getCount())
-        return false;
-
-    out.origin = world->engravings[index];
-
-    out.artist = out.origin->artist;
-    out.masterpiece_event = out.origin->masterpiece_event;
-    out.skill_rating = out.origin->skill_rating;
-    out.pos = out.origin->pos;
-    out.flags = out.origin->flags;
-    out.tile = out.origin->tile;
-    out.art_id = out.origin->art_id;
-    out.art_subid = out.origin->art_subid;
-    out.quality = out.origin->quality;
-    return true;
-}

library/modules/Filesystem.cpp

@@ -246,6 +246,7 @@ static int listdir_recursive_impl (std::string prefix, std::string path,
     int err = Filesystem::listdir(prefixed_path, curdir_files);
     if (err)
         return err;
+    bool out_of_depth = false;
     for (auto file = curdir_files.begin(); file != curdir_files.end(); ++file)
     {
         if (*file == "." || *file == "..")
@@ -254,6 +255,12 @@ static int listdir_recursive_impl (std::string prefix, std::string path,
         std::string path_file = path + *file;
         if (Filesystem::isdir(prefixed_file))
         {
+            if (depth == 0)
+            {
+                out_of_depth = true;
+                continue;
+            }
+
             files.insert(std::pair<std::string, bool>(include_prefix ? prefixed_file : path_file, true));
             err = listdir_recursive_impl(prefix, path_file + "/", files, depth - 1, include_prefix);
             if (err)
@@ -264,7 +271,7 @@ static int listdir_recursive_impl (std::string prefix, std::string path,
             files.insert(std::pair<std::string, bool>(include_prefix ? prefixed_file : path_file, false));
         }
     }
-    return 0;
+    return out_of_depth ? -1 : 0;
 }
 
 int Filesystem::listdir_recursive (std::string dir, std::map<std::string, bool> &files,

library/modules/Gui.cpp

@@ -106,6 +106,7 @@ using namespace DFHack;
 #include "df/viewscreen_unitlistst.h"
 #include "df/viewscreen_unitst.h"
 #include "df/viewscreen_reportlistst.h"
+#include "df/viewscreen_treasurelistst.h"
 #include "df/viewscreen_workquota_conditionst.h"
 #include "df/viewscreen_workshop_profilest.h"
 #include "df/world.h"
@@ -1181,6 +1182,13 @@ df::item *Gui::getAnyItem(df::viewscreen *top)
         return NULL;
     }
 
+    if (VIRTUAL_CAST_VAR(screen, df::viewscreen_treasurelistst, top))
+    {
+        if (world)
+            return vector_get(world->items.other[df::items_other_id::ANY_ARTIFACT], screen->sel_idx);
+        return NULL;
+    }
+
     if (auto dfscreen = dfhack_viewscreen::try_cast(top))
         return dfscreen->getSelectedItem();
 

library/modules/Job.cpp

@@ -360,27 +360,24 @@ bool DFHack::Job::removeJob(df::job* job) {
     using df::global::world;
     CHECK_NULL_POINTER(job);
 
-    // cancel_job below does not clean up refs, so we have to do that first
+    // cancel_job below does not clean up all refs, so we have to do some work
 
-    // clean up general refs
+    // manually handle DESTROY_BUILDING jobs (cancel_job doesn't handle them)
-    for (auto genRef : job->general_refs) {
+    if (job->job_type == df::job_type::DestroyBuilding) {
-        if (!genRef) continue;
+        for (auto &genRef : job->general_refs) {
-
+            disconnectJobGeneralRef(job, genRef);
-        // disconnectJobGeneralRef only handles buildings and units
-        if (genRef->getType() != general_ref_type::BUILDING_HOLDER &&
-                genRef->getType() != general_ref_type::UNIT_WORKER)
-            return false;
-    }
-
-    for (auto genRef : job->general_refs) {
-        // this should always succeed because of the check in the preceding loop
-        bool success = disconnectJobGeneralRef(job, genRef);
-        assert(success); (void)success;
             if (genRef) delete genRef;
         }
         job->general_refs.resize(0);
 
-    // clean up item refs
+        // remove the job from the world
+        job->list_link->prev->next = job->list_link->next;
+        delete job->list_link;
+        delete job;
+        return true;
+    }
+
+    // clean up item refs and delete them
     for (auto &item_ref : job->items) {
        disconnectJobItem(job, item_ref);
        if (item_ref) delete item_ref;

library/modules/Notes.cpp

@@ -1,53 +0,0 @@
-/*
-www.sourceforge.net/projects/dfhack
-Copyright (c) 2009 Petr Mrázek (peterix), Kenneth Ferland (Impaler[WrG]), dorf
-
-This software is provided 'as-is', without any express or implied
-warranty. In no event will the authors be held liable for any
-damages arising from the use of this software.
-
-Permission is granted to anyone to use this software for any
-purpose, including commercial applications, and to alter it and
-redistribute it freely, subject to the following restrictions:
-
-1. The origin of this software must not be misrepresented; you must
-not claim that you wrote the original software. If you use this
-software in a product, an acknowledgment in the product documentation
-would be appreciated but is not required.
-
-2. Altered source versions must be plainly marked as such, and
-must not be misrepresented as being the original software.
-
-3. This notice may not be removed or altered from any source
-distribution.
-*/
-
-#include "Internal.h"
-
-#include <string>
-#include <vector>
-#include <map>
-using namespace std;
-
-#include "VersionInfo.h"
-#include "Types.h"
-#include "Error.h"
-#include "MemAccess.h"
-#include "MiscUtils.h"
-#include "ModuleFactory.h"
-#include "Core.h"
-#include "modules/Notes.h"
-#include <DataDefs.h>
-#include "df/ui.h"
-using namespace DFHack;
-
-std::unique_ptr<Module> DFHack::createNotes()
-{
-    return dts::make_unique<Notes>();
-}
-
-// FIXME: not even a wrapper now
-Notes::Notes()
-{
-    notes = (std::vector<t_note*>*) &df::global::ui->waypoints.points;
-}

library/modules/Windows.cpp

@@ -1,118 +0,0 @@
-/*
-https://github.com/peterix/dfhack
-Copyright (c) 2009-2012 Petr Mrázek (peterix@gmail.com)
-
-This software is provided 'as-is', without any express or implied
-warranty. In no event will the authors be held liable for any
-damages arising from the use of this software.
-
-Permission is granted to anyone to use this software for any
-purpose, including commercial applications, and to alter it and
-redistribute it freely, subject to the following restrictions:
-
-1. The origin of this software must not be misrepresented; you must
-not claim that you wrote the original software. If you use this
-software in a product, an acknowledgment in the product documentation
-would be appreciated but is not required.
-
-2. Altered source versions must be plainly marked as such, and
-must not be misrepresented as being the original software.
-
-3. This notice may not be removed or altered from any source
-distribution.
-*/
-
-#include "Export.h"
-#include "Module.h"
-#include "BitArray.h"
-#include <string>
-
-#include "DataDefs.h"
-#include "df/init.h"
-#include "df/ui.h"
-#include <df/graphic.h>
-#include "modules/Windows.h"
-
-using namespace DFHack;
-using df::global::gps;
-
-Windows::df_screentile *Windows::getScreenBuffer()
-{
-    if (!gps) return NULL;
-    return (df_screentile *) gps->screen;
-}
-
-Windows::df_window::df_window(int x, int y, unsigned int width, unsigned int height)
-:buffer(0), width(width), height(height), parent(0), left(x), top(y), current_painter(NULL)
-{
-    buffer = 0;
-};
-Windows::df_window::~df_window()
-{
-    for(auto iter = children.begin();iter != children.end();iter++)
-    {
-        delete *iter;
-    }
-    children.clear();
-};
-Windows::painter * Windows::df_window::lock()
-{
-    locked = true;
-    current_painter = new Windows::painter(this,buffer,width, height);
-    return current_painter;
-};
-
-bool Windows::df_window::addChild( df_window * child)
-{
-    children.push_back(child);
-    child->parent = this;
-    return true;
-}
-
-bool Windows::df_window::unlock (painter * painter)
-{
-    if(current_painter == painter)
-    {
-        delete current_painter;
-        current_painter = 0;
-        locked = false;
-        return true;
-    }
-    return false;
-}
-
-Windows::top_level_window::top_level_window() : df_window(0,0,gps ? gps->dimx : 80,gps ? gps->dimy : 25)
-{
-    buffer = 0;
-}
-
-bool Windows::top_level_window::move (int left_, int top_, unsigned int width_, unsigned int height_)
-{
-    width = width_;
-    height = height_;
-    // what if we are painting already? Is that possible?
-    return true;
-};
-
-Windows::painter * Windows::top_level_window::lock()
-{
-    buffer = getScreenBuffer();
-    return df_window::lock();
-}
-
-void Windows::top_level_window::paint ()
-{
-    for(auto iter = children.begin();iter != children.end();iter++)
-    {
-        (*iter)->paint();
-    }
-};
-
-Windows::df_tilebuf Windows::top_level_window::getBuffer()
-{
-    df_tilebuf buf;
-    buf.data = getScreenBuffer();
-    buf.height = df::global::gps->dimy;
-    buf.width = df::global::gps->dimx;
-    return buf;
-}

library/xml

@@ -1 +1 @@
-Subproject commit ef5c180ff220e53ec6b51e01ca3594a5780b8b4b
+Subproject commit dc118c5e90aea6181a290e4bbf40e7f2974fb053

plugins/CMakeLists.txt

@@ -83,6 +83,7 @@ if(BUILD_SUPPORTED)
     dfhack_plugin(3dveins 3dveins.cpp)
     dfhack_plugin(add-spatter add-spatter.cpp)
     # dfhack_plugin(advtools advtools.cpp)
+    dfhack_plugin(autobutcher autobutcher.cpp LINK_LIBRARIES lua)
     dfhack_plugin(autochop autochop.cpp)
     dfhack_plugin(autoclothing autoclothing.cpp)
     dfhack_plugin(autodump autodump.cpp)
@@ -92,6 +93,7 @@ if(BUILD_SUPPORTED)
     dfhack_plugin(autolabor autolabor.cpp)
     dfhack_plugin(automaterial automaterial.cpp LINK_LIBRARIES lua)
     dfhack_plugin(automelt automelt.cpp)
+    dfhack_plugin(autonestbox autonestbox.cpp LINK_LIBRARIES lua)
     dfhack_plugin(autotrade autotrade.cpp)
     dfhack_plugin(blueprint blueprint.cpp LINK_LIBRARIES lua)
     dfhack_plugin(burrows burrows.cpp LINK_LIBRARIES lua)
@@ -148,7 +150,7 @@ if(BUILD_SUPPORTED)
     dfhack_plugin(petcapRemover petcapRemover.cpp)
     dfhack_plugin(plants plants.cpp)
     dfhack_plugin(probe probe.cpp)
-    dfhack_plugin(prospector prospector.cpp)
+    dfhack_plugin(prospector prospector.cpp LINK_LIBRARIES lua)
     dfhack_plugin(power-meter power-meter.cpp LINK_LIBRARIES lua)
     dfhack_plugin(regrass regrass.cpp)
     add_subdirectory(remotefortressreader)
@@ -177,7 +179,7 @@ if(BUILD_SUPPORTED)
     dfhack_plugin(workflow workflow.cpp LINK_LIBRARIES lua)
     dfhack_plugin(workNow workNow.cpp)
     dfhack_plugin(xlsxreader xlsxreader.cpp LINK_LIBRARIES lua xlsxio_read_STATIC zip expat)
-    dfhack_plugin(zone zone.cpp LINK_LIBRARIES lua)
+    dfhack_plugin(zone zone.cpp)
 
     # If you are adding a plugin that you do not intend to commit to the DFHack repo,
     # see instructions for adding "external" plugins at the end of this file.
@@ -186,7 +188,7 @@ endif()
 # this is the skeleton plugin. If you want to make your own, make a copy and then change it
 option(BUILD_SKELETON "Build the skeleton plugin." OFF)
 if(BUILD_SKELETON)
-    add_subdirectory(skeleton)
+    dfhack_plugin(skeleton examples/skeleton.cpp)
 endif()
 
 macro(subdirlist result subdir)

plugins/automelt.cpp

@@ -250,19 +250,6 @@ IMPLEMENT_VMETHOD_INTERPOSE(melt_hook, feed);
 IMPLEMENT_VMETHOD_INTERPOSE(melt_hook, render);
 
 
-static command_result automelt_cmd(color_ostream &out, vector <string> & parameters)
-{
-    if (!parameters.empty())
-    {
-        if (parameters.size() == 1 && toLower(parameters[0])[0] == 'v')
-        {
-            out << "Automelt" << endl << "Version: " << PLUGIN_VERSION << endl;
-        }
-    }
-
-    return CR_OK;
-}
-
 DFhackCExport command_result plugin_onstatechange(color_ostream &out, state_change_event event)
 {
     switch (event)
@@ -296,15 +283,12 @@ DFhackCExport command_result plugin_enable(color_ostream &out, bool enable)
 
 DFhackCExport command_result plugin_init ( color_ostream &out, std::vector <PluginCommand> &commands)
 {
-    commands.push_back(
-        PluginCommand(
-        "automelt", "Automatically melt metal items in marked stockpiles.",
-        automelt_cmd, false, ""));
-
     return CR_OK;
 }
 
 DFhackCExport command_result plugin_shutdown ( color_ostream &out )
 {
+    // ensure we disengage our hooks
+    plugin_enable(out, false);
     return CR_OK;
 }

plugins/autonestbox.cpp

@@ -0,0 +1,430 @@
+// - full automation of handling mini-pastures over nestboxes:
+//   go through all pens, check if they are empty and placed over a nestbox
+//   find female tame egg-layer who is not assigned to another pen and assign it to nestbox pasture
+//   maybe check for minimum age? it's not that useful to fill nestboxes with freshly hatched birds
+//   state and sleep setting is saved the first time autonestbox is started (to avoid writing stuff if the plugin is never used)
+
+#include <string>
+#include <vector>
+
+#include "df/building_cagest.h"
+#include "df/building_civzonest.h"
+#include "df/building_nest_boxst.h"
+#include "df/general_ref_building_civzone_assignedst.h"
+#include "df/world.h"
+
+#include "Debug.h"
+#include "LuaTools.h"
+#include "PluginManager.h"
+
+#include "modules/Buildings.h"
+#include "modules/Gui.h"
+#include "modules/Persistence.h"
+#include "modules/Units.h"
+#include "modules/World.h"
+
+using std::string;
+using std::vector;
+
+using namespace DFHack;
+
+DFHACK_PLUGIN("autonestbox");
+DFHACK_PLUGIN_IS_ENABLED(is_enabled);
+
+REQUIRE_GLOBAL(world);
+
+static const string autonestbox_help =
+    "Assigns unpastured female egg-layers to nestbox zones.\n"
+    "Requires that you create pen/pasture zones above nestboxes.\n"
+    "If the pen is bigger than 1x1 the nestbox must be in the top left corner.\n"
+    "Only 1 unit will be assigned per pen, regardless of the size.\n"
+    "The age of the units is currently not checked, most birds grow up quite fast.\n"
+    "Usage:\n"
+    "\n"
+    "enable autonestbox\n"
+    "    Start checking for unpastured egg-layers and assigning them to nestbox zones.\n"
+    "autonestbox\n"
+    "    Print current status."
+    "autonestbox now\n"
+    "    Run a scan and assignment cycle right now. Does not require that the plugin is enabled.\n"
+    "autonestbox ticks <ticks>\n"
+    "    Change the number of ticks between scan and assignment cycles when the plugin is enabled.\n"
+    "    The default is 6000 (about 8 days)\n";
+
+namespace DFHack {
+    // for configuration-related logging
+    DBG_DECLARE(autonestbox, status, DebugCategory::LINFO);
+    // for logging during the periodic scan
+    DBG_DECLARE(autonestbox, cycle, DebugCategory::LINFO);
+}
+
+static const string CONFIG_KEY = string(plugin_name) + "/config";
+static PersistentDataItem config;
+enum ConfigValues {
+    CONFIG_IS_ENABLED = 0,
+    CONFIG_CYCLE_TICKS = 1,
+};
+static int get_config_val(int index) {
+    if (!config.isValid())
+        return -1;
+    return config.ival(index);
+}
+static bool get_config_bool(int index) {
+    return get_config_val(index) == 1;
+}
+static void set_config_val(int index, int value) {
+    if (config.isValid())
+        config.ival(index) = value;
+}
+static void set_config_bool(int index, bool value) {
+    set_config_val(index, value ? 1 : 0);
+}
+
+static bool did_complain = false; // avoids message spam
+static int32_t cycle_timestamp = 0;  // world->frame_counter at last cycle
+
+static command_result df_autonestbox(color_ostream &out, vector<string> &parameters);
+static void autonestbox_cycle(color_ostream &out);
+
+DFhackCExport command_result plugin_init(color_ostream &out, std::vector <PluginCommand> &commands) {
+    commands.push_back(PluginCommand(
+        plugin_name,
+        "Auto-assign egg-laying female pets to nestbox zones.",
+        df_autonestbox,
+        false,
+        autonestbox_help.c_str()));
+    return CR_OK;
+}
+
+DFhackCExport command_result plugin_enable(color_ostream &out, bool enable) {
+    if (!Core::getInstance().isWorldLoaded()) {
+        out.printerr("Cannot enable %s without a loaded world.\n", plugin_name);
+        return CR_FAILURE;
+    }
+
+    if (enable != is_enabled) {
+        is_enabled = enable;
+        DEBUG(status,out).print("%s from the API; persisting\n",
+                                is_enabled ? "enabled" : "disabled");
+        set_config_bool(CONFIG_IS_ENABLED, is_enabled);
+    } else {
+        DEBUG(status,out).print("%s from the API, but already %s; no action\n",
+                                is_enabled ? "enabled" : "disabled",
+                                is_enabled ? "enabled" : "disabled");
+    }
+    return CR_OK;
+}
+
+DFhackCExport command_result plugin_load_data (color_ostream &out) {
+    config = World::GetPersistentData(CONFIG_KEY);
+
+    if (!config.isValid()) {
+        DEBUG(status,out).print("no config found in this save; initializing\n");
+        config = World::AddPersistentData(CONFIG_KEY);
+        set_config_bool(CONFIG_IS_ENABLED, is_enabled);
+        set_config_val(CONFIG_CYCLE_TICKS, 6000);
+    }
+
+    // we have to copy our enabled flag into the global plugin variable, but
+    // all the other state we can directly read/modify from the persistent
+    // data structure.
+    is_enabled = get_config_bool(CONFIG_IS_ENABLED);
+    DEBUG(status,out).print("loading persisted enabled state: %s\n",
+                            is_enabled ? "true" : "false");
+    did_complain = false;
+    return CR_OK;
+}
+
+DFhackCExport command_result plugin_onstatechange(color_ostream &out, state_change_event event) {
+    if (event == DFHack::SC_WORLD_UNLOADED) {
+        if (is_enabled) {
+            DEBUG(status,out).print("world unloaded; disabling %s\n",
+                                    plugin_name);
+            is_enabled = false;
+        }
+    }
+    return CR_OK;
+}
+
+DFhackCExport command_result plugin_onupdate(color_ostream &out) {
+    if (is_enabled && world->frame_counter - cycle_timestamp >= get_config_val(CONFIG_CYCLE_TICKS))
+        autonestbox_cycle(out);
+    return CR_OK;
+}
+
+/////////////////////////////////////////////////////
+// configuration interface
+//
+
+struct autonestbox_options {
+    // whether to display help
+    bool help = false;
+
+    // whether to run a cycle right now
+    bool now = false;
+
+    // how many ticks to wait between automatic cycles, -1 means unset
+    int32_t ticks = -1;
+
+    static struct_identity _identity;
+};
+static const struct_field_info autonestbox_options_fields[] = {
+    { struct_field_info::PRIMITIVE, "help",  offsetof(autonestbox_options, help),  &df::identity_traits<bool>::identity,    0, 0 },
+    { struct_field_info::PRIMITIVE, "now",   offsetof(autonestbox_options, now),   &df::identity_traits<bool>::identity,    0, 0 },
+    { struct_field_info::PRIMITIVE, "ticks", offsetof(autonestbox_options, ticks), &df::identity_traits<int32_t>::identity, 0, 0 },
+    { struct_field_info::END }
+};
+struct_identity autonestbox_options::_identity(sizeof(autonestbox_options), &df::allocator_fn<autonestbox_options>, NULL, "autonestbox_options", NULL, autonestbox_options_fields);
+
+static bool get_options(color_ostream &out,
+                        autonestbox_options &opts,
+                        const vector<string> &parameters)
+{
+    auto L = Lua::Core::State;
+    Lua::StackUnwinder top(L);
+
+    if (!lua_checkstack(L, parameters.size() + 2) ||
+        !Lua::PushModulePublic(
+            out, L, "plugins.autonestbox", "parse_commandline")) {
+        out.printerr("Failed to load autonestbox Lua code\n");
+        return false;
+    }
+
+    Lua::Push(L, &opts);
+    for (const string &param : parameters)
+        Lua::Push(L, param);
+
+    if (!Lua::SafeCall(out, L, parameters.size() + 1, 0))
+        return false;
+
+    return true;
+}
+
+static command_result df_autonestbox(color_ostream &out, vector<string> &parameters) {
+    CoreSuspender suspend;
+
+    if (!Core::getInstance().isWorldLoaded()) {
+        out.printerr("Cannot run %s without a loaded world.\n", plugin_name);
+        return CR_FAILURE;
+    }
+
+    autonestbox_options opts;
+    if (!get_options(out, opts, parameters) || opts.help)
+        return CR_WRONG_USAGE;
+
+    if (opts.ticks > -1) {
+        set_config_val(CONFIG_CYCLE_TICKS, opts.ticks);
+        INFO(status,out).print("New cycle timer: %d ticks.\n", opts.ticks);
+    }
+    else if (opts.now) {
+        autonestbox_cycle(out);
+    }
+    else {
+        out << "autonestbox is " << (is_enabled ? "" : "not ") << "running" << endl;
+    }
+    return CR_OK;
+}
+
+/////////////////////////////////////////////////////
+// cycle logic
+//
+
+static bool isEmptyPasture(df::building *building) {
+    if (!Buildings::isPenPasture(building))
+        return false;
+    df::building_civzonest *civ = (df::building_civzonest *)building;
+    return (civ->assigned_units.size() == 0);
+}
+
+static bool isFreeNestboxAtPos(int32_t x, int32_t y, int32_t z) {
+    for (auto building : world->buildings.all) {
+        if (building->getType() == df::building_type::NestBox
+                && building->x1 == x
+                && building->y1 == y
+                && building->z  == z) {
+            df::building_nest_boxst *nestbox = (df::building_nest_boxst *)building;
+            if (nestbox->claimed_by == -1 && nestbox->contained_items.size() == 1) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
+static df::building* findFreeNestboxZone() {
+    for (auto building : world->buildings.all) {
+        if (isEmptyPasture(building) &&
+                Buildings::isActive(building) &&
+                isFreeNestboxAtPos(building->x1, building->y1, building->z)) {
+            return building;
+        }
+    }
+    return NULL;
+}
+
+static bool isInBuiltCage(df::unit *unit) {
+    for (auto building : world->buildings.all) {
+        if (building->getType() == df::building_type::Cage) {
+            df::building_cagest* cage = (df::building_cagest *)building;
+            for (auto unitid : cage->assigned_units) {
+                if (unitid == unit->id)
+                    return true;
+            }
+        }
+    }
+    return false;
+}
+
+// check if assigned to pen, pit, (built) cage or chain
+// note: BUILDING_CAGED is not set for animals (maybe it's used for dwarves who get caged as sentence)
+// animals in cages (no matter if built or on stockpile) get the ref CONTAINED_IN_ITEM instead
+// removing them from cages on stockpiles is no problem even without clearing the ref
+// and usually it will be desired behavior to do so.
+static bool isAssigned(df::unit *unit) {
+    for (auto ref : unit->general_refs) {
+        auto rtype = ref->getType();
+        if(rtype == df::general_ref_type::BUILDING_CIVZONE_ASSIGNED
+                || rtype == df::general_ref_type::BUILDING_CAGED
+                || rtype == df::general_ref_type::BUILDING_CHAIN
+                || (rtype == df::general_ref_type::CONTAINED_IN_ITEM && isInBuiltCage(unit))) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static bool isFreeEgglayer(df::unit *unit)
+{
+    return Units::isActive(unit) && !Units::isUndead(unit)
+        && Units::isFemale(unit)
+        && Units::isTame(unit)
+        && Units::isOwnCiv(unit)
+        && Units::isEggLayer(unit)
+        && !isAssigned(unit)
+        && !Units::isGrazer(unit) // exclude grazing birds because they're messy
+        && !Units::isMerchant(unit) // don't steal merchant mounts
+        && !Units::isForest(unit);  // don't steal birds from traders, they hate that
+}
+
+static df::unit * findFreeEgglayer() {
+    for (auto unit : world->units.all) {
+        if (isFreeEgglayer(unit))
+            return unit;
+    }
+    return NULL;
+}
+
+static df::general_ref_building_civzone_assignedst * createCivzoneRef() {
+    static bool vt_initialized = false;
+
+    // after having run successfully for the first time it's safe to simply create the object
+    if (vt_initialized) {
+        return (df::general_ref_building_civzone_assignedst *)
+            df::general_ref_building_civzone_assignedst::_identity.instantiate();
+    }
+
+    // being called for the first time, need to initialize the vtable
+    for (auto creature : world->units.all) {
+        for (auto ref : creature->general_refs) {
+            if (ref->getType() == df::general_ref_type::BUILDING_CIVZONE_ASSIGNED) {
+                if (strict_virtual_cast<df::general_ref_building_civzone_assignedst>(ref)) {
+                    vt_initialized = true;
+                    // !! calling new() doesn't work, need _identity.instantiate() instead !!
+                    return (df::general_ref_building_civzone_assignedst *)
+                        df::general_ref_building_civzone_assignedst::_identity.instantiate();
+                }
+            }
+        }
+    }
+    return NULL;
+}
+
+static bool assignUnitToZone(color_ostream &out, df::unit *unit, df::building *building) {
+    // try to get a fresh civzone ref
+    df::general_ref_building_civzone_assignedst *ref = createCivzoneRef();
+    if (!ref) {
+        ERR(cycle,out).print("Could not find a clonable activity zone reference!"
+            " You need to manually pen/pasture/pit at least one creature"
+            " before autonestbox can function.\n");
+        return false;
+    }
+
+    ref->building_id = building->id;
+    unit->general_refs.push_back(ref);
+
+    df::building_civzonest *civz = (df::building_civzonest *)building;
+    civz->assigned_units.push_back(unit->id);
+
+    INFO(cycle,out).print("Unit %d (%s) assigned to nestbox zone %d (%s)\n",
+        unit->id, Units::getRaceName(unit).c_str(),
+        building->id, building->name.c_str());
+
+    return true;
+}
+
+static size_t countFreeEgglayers() {
+    size_t count = 0;
+    for (auto unit : world->units.all) {
+        if (isFreeEgglayer(unit))
+            ++count;
+    }
+    return count;
+}
+
+static size_t assign_nestboxes(color_ostream &out) {
+    size_t processed = 0;
+    df::building *free_building = NULL;
+    df::unit *free_unit = NULL;
+    do {
+        free_building = findFreeNestboxZone();
+        free_unit = findFreeEgglayer();
+        if (free_building && free_unit) {
+            if (!assignUnitToZone(out, free_unit, free_building)) {
+                DEBUG(cycle,out).print("Failed to assign unit to building.\n");
+                return processed;
+            }
+            DEBUG(cycle,out).print("assigned unit %d to zone %d\n",
+                                   free_unit->id, free_building->id);
+            ++processed;
+        }
+    } while (free_unit && free_building);
+
+    if (free_unit && !free_building) {
+        static size_t old_count = 0;
+        size_t freeEgglayers = countFreeEgglayers();
+        // avoid spamming the same message
+        if (old_count != freeEgglayers)
+            did_complain = false;
+        old_count = freeEgglayers;
+        if (!did_complain) {
+            stringstream ss;
+            ss << freeEgglayers;
+            string announce = "Not enough free nestbox zones found! You need " + ss.str() + " more.";
+            Gui::showAnnouncement(announce, 6, true);
+            out << announce << endl;
+            did_complain = true;
+        }
+    }
+    return processed;
+}
+
+static void autonestbox_cycle(color_ostream &out) {
+    // mark that we have recently run
+    cycle_timestamp = world->frame_counter;
+
+    DEBUG(cycle,out).print("running autonestbox cycle\n");
+
+    size_t processed = assign_nestboxes(out);
+    if (processed > 0) {
+        stringstream ss;
+        ss << processed << " nestboxes were assigned.";
+        string announce = ss.str();
+        DEBUG(cycle,out).print("%s\n", announce.c_str());
+        Gui::showAnnouncement(announce, 2, false);
+        out << announce << endl;
+        // can complain again
+        // (might lead to spamming the same message twice, but catches the case
+        // where for example 2 new egglayers hatched right after 2 zones were created and assigned)
+        did_complain = false;
+    }
+}

plugins/building-hacks.cpp

@@ -88,8 +88,8 @@ struct work_hook : df::building_workshopst{
             df::general_ref_creaturest* ref = static_cast<df::general_ref_creaturest*>(DFHack::Buildings::getGeneralRef(this, general_ref_type::CREATURE));
             if (ref)
             {
-                info->produced = ref->anon_1;
+                info->produced = ref->unk_1;
-                info->consumed = ref->anon_2;
+                info->consumed = ref->unk_2;
                 return true;
             }
             else
@@ -118,14 +118,14 @@ struct work_hook : df::building_workshopst{
         df::general_ref_creaturest* ref = static_cast<df::general_ref_creaturest*>(DFHack::Buildings::getGeneralRef(this, general_ref_type::CREATURE));
         if (ref)
         {
-            ref->anon_1 = produced;
+            ref->unk_1 = produced;
-            ref->anon_2 = consumed;
+            ref->unk_2 = consumed;
         }
         else
         {
             ref = df::allocate<df::general_ref_creaturest>();
-            ref->anon_1 = produced;
+            ref->unk_1 = produced;
-            ref->anon_2 = consumed;
+            ref->unk_2 = consumed;
             general_refs.push_back(ref);
         }
     }

plugins/devel/CMakeLists.txt

@@ -13,7 +13,6 @@ dfhack_plugin(eventExample eventExample.cpp)
 dfhack_plugin(frozen frozen.cpp)
 dfhack_plugin(kittens kittens.cpp LINK_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})
 dfhack_plugin(memview memview.cpp memutils.cpp LINK_LIBRARIES lua)
-dfhack_plugin(notes notes.cpp)
 dfhack_plugin(onceExample onceExample.cpp)
 dfhack_plugin(renderer-msg renderer-msg.cpp)
 dfhack_plugin(rprobe rprobe.cpp)

plugins/devel/notes.cpp

@@ -1,72 +0,0 @@
-#include "Core.h"
-#include <Console.h>
-#include <Export.h>
-#include <PluginManager.h>
-#include <vector>
-#include <string>
-#include <modules/Notes.h>
-
-using std::vector;
-using std::string;
-using namespace DFHack;
-
-command_result df_notes (color_ostream &out, vector <string> & parameters);
-
-DFHACK_PLUGIN("notes");
-
-DFhackCExport command_result plugin_init ( color_ostream &out, std::vector <PluginCommand> &commands)
-{
-    commands.push_back(PluginCommand("dumpnotes",
-               "Dumps in-game notes",
-               df_notes));
-    return CR_OK;
-}
-
-DFhackCExport command_result plugin_shutdown ( color_ostream &out )
-{
-    return CR_OK;
-}
-
-command_result df_notes (color_ostream &con, vector <string> & parameters)
-{
-    CoreSuspender suspend;
-
-    DFHack::Notes * note_mod = Core::getInstance().getNotes();
-    std::vector<t_note*>* note_list = note_mod->notes;
-
-    if (note_list == NULL)
-    {
-        con.printerr("Notes are not supported under this version of DF.\n");
-        return CR_OK;
-    }
-
-    if (note_list->empty())
-    {
-        con << "There are no notes." << std::endl;
-        return CR_OK;
-    }
-
-
-    for (size_t i = 0; i < note_list->size(); i++)
-    {
-        t_note* note = (*note_list)[i];
-
-        con.print("Note %p at: %d/%d/%d\n",note, note->x, note->y, note->z);
-        con.print("Note id: %d\n", note->id);
-        con.print("Note symbol: '%c'\n", note->symbol);
-
-        if (note->name.length() > 0)
-            con << "Note name: " << (note->name) << std::endl;
-        if (note->text.length() > 0)
-            con << "Note text: " << (note->text) << std::endl;
-
-        if (note->unk1 != 0)
-            con.print("unk1: %x\n", note->unk1);
-        if (note->unk2 != 0)
-            con.print("unk2: %x\n", note->unk2);
-
-        con << std::endl;
-    }
-
-    return CR_OK;
-}

plugins/devel/renderer-msg.cpp

@@ -23,13 +23,33 @@ struct scdata {
 };
 
 struct renderer_msg : public Renderer::renderer_wrap {
-    virtual void update_tile (int32_t x, int32_t y) {
+    bool message_dirty = true;  // force redraw when renderer is installed
+
+    virtual void update_tile(int32_t x, int32_t y) override {
+        draw_message();
+        renderer_wrap::update_tile(x, y);
+    }
+
+    virtual void update_all() override {
+        draw_message();
+        renderer_wrap::update_all();
+    }
+
+    virtual void render() override {
+        message_dirty = true;
+        renderer_wrap::render();
+    }
+
+    void draw_message() {
+        if (message_dirty) {
             static std::string str = std::string("DFHack: ") + plugin_name + " active";
-        Screen::paintString(Screen::Pen(' ', 9, 0), 0, gps->dimy - 1, str);
+            Screen::paintString(Screen::Pen(' ', COLOR_LIGHTCYAN, COLOR_GREEN), 0, gps->dimy - 1, str);
             for (int32_t i = 0; i < gps->dimx; ++i)
                 ((scdata*)screen)[i * gps->dimy + gps->dimy - 1].bg = 2;
-        renderer_wrap::update_tile(x, y);
+
-    };
+            message_dirty = false;
+        }
+    }
 };
 
 DFhackCExport command_result plugin_init (color_ostream &out, std::vector <PluginCommand> &commands)

plugins/examples/persistent_per_save_example.cpp

@@ -0,0 +1,166 @@
+// This template is appropriate for plugins that periodically check game state
+// and make some sort of automated change. These types of plugins typically
+// provide a command that can be used to configure the plugin behavior and
+// require a world to be loaded before they can function. This kind of plugin
+// should persist its state in the savegame and auto-re-enable itself when a
+// savegame that had this plugin enabled is loaded.
+
+#include <string>
+#include <vector>
+
+#include "df/world.h"
+
+#include "Core.h"
+#include "Debug.h"
+#include "PluginManager.h"
+
+#include "modules/Persistence.h"
+#include "modules/World.h"
+
+using std::string;
+using std::vector;
+
+using namespace DFHack;
+
+DFHACK_PLUGIN("persistent_per_save_example");
+DFHACK_PLUGIN_IS_ENABLED(is_enabled);
+
+REQUIRE_GLOBAL(world);
+
+// logging levels can be dynamically controlled with the `debugfilter` command.
+namespace DFHack {
+    // for configuration-related logging
+    DBG_DECLARE(persistent_per_save_example, status, DebugCategory::LINFO);
+    // for logging during the periodic scan
+    DBG_DECLARE(persistent_per_save_example, cycle, DebugCategory::LINFO);
+}
+
+static const string CONFIG_KEY = string(plugin_name) + "/config";
+static PersistentDataItem config;
+enum ConfigValues {
+    CONFIG_IS_ENABLED = 0,
+    CONFIG_CYCLE_TICKS = 1,
+};
+static int get_config_val(int index) {
+    if (!config.isValid())
+        return -1;
+    return config.ival(index);
+}
+static bool get_config_bool(int index) {
+    return get_config_val(index) == 1;
+}
+static void set_config_val(int index, int value) {
+    if (config.isValid())
+        config.ival(index) = value;
+}
+static void set_config_bool(int index, bool value) {
+    set_config_val(index, value ? 1 : 0);
+}
+
+static int32_t cycle_timestamp = 0;  // world->frame_counter at last cycle
+
+static command_result do_command(color_ostream &out, vector<string> &parameters);
+static void do_cycle(color_ostream &out);
+
+DFhackCExport command_result plugin_init(color_ostream &out, std::vector <PluginCommand> &commands) {
+    DEBUG(status,out).print("initializing %s\n", plugin_name);
+
+    // provide a configuration interface for the plugin
+    commands.push_back(PluginCommand(
+        plugin_name,
+        "Short (~54 character) description of command.",
+        do_command));
+
+    return CR_OK;
+}
+
+DFhackCExport command_result plugin_enable(color_ostream &out, bool enable) {
+    if (!Core::getInstance().isWorldLoaded()) {
+        out.printerr("Cannot enable %s without a loaded world.\n", plugin_name);
+        return CR_FAILURE;
+    }
+
+    if (enable != is_enabled) {
+        is_enabled = enable;
+        DEBUG(status,out).print("%s from the API; persisting\n",
+                                is_enabled ? "enabled" : "disabled");
+        set_config_bool(CONFIG_IS_ENABLED, is_enabled);
+    } else {
+        DEBUG(status,out).print("%s from the API, but already %s; no action\n",
+                                is_enabled ? "enabled" : "disabled",
+                                is_enabled ? "enabled" : "disabled");
+    }
+    return CR_OK;
+}
+
+DFhackCExport command_result plugin_shutdown (color_ostream &out) {
+    DEBUG(status,out).print("shutting down %s\n", plugin_name);
+
+    return CR_OK;
+}
+
+DFhackCExport command_result plugin_load_data (color_ostream &out) {
+    config = World::GetPersistentData(CONFIG_KEY);
+
+    if (!config.isValid()) {
+        DEBUG(status,out).print("no config found in this save; initializing\n");
+        config = World::AddPersistentData(CONFIG_KEY);
+        set_config_bool(CONFIG_IS_ENABLED, is_enabled);
+        set_config_val(CONFIG_CYCLE_TICKS, 6000);
+    }
+
+    // we have to copy our enabled flag into the global plugin variable, but
+    // all the other state we can directly read/modify from the persistent
+    // data structure.
+    is_enabled = get_config_bool(CONFIG_IS_ENABLED);
+    DEBUG(status,out).print("loading persisted enabled state: %s\n",
+                            is_enabled ? "true" : "false");
+    return CR_OK;
+}
+
+DFhackCExport command_result plugin_onstatechange(color_ostream &out, state_change_event event) {
+    if (event == DFHack::SC_WORLD_UNLOADED) {
+        if (is_enabled) {
+            DEBUG(status,out).print("world unloaded; disabling %s\n",
+                                    plugin_name);
+            is_enabled = false;
+        }
+    }
+    return CR_OK;
+}
+
+DFhackCExport command_result plugin_onupdate(color_ostream &out) {
+    if (is_enabled && world->frame_counter - cycle_timestamp >= get_config_val(CONFIG_CYCLE_TICKS))
+        do_cycle(out);
+    return CR_OK;
+}
+
+static command_result do_command(color_ostream &out, vector<string> &parameters) {
+    // be sure to suspend the core if any DF state is read or modified
+    CoreSuspender suspend;
+
+    if (!Core::getInstance().isWorldLoaded()) {
+        out.printerr("Cannot run %s without a loaded world.\n", plugin_name);
+        return CR_FAILURE;
+    }
+
+    // TODO: configuration logic
+    // simple commandline parsing can be done in C++, but there are lua libraries
+    // that can easily handle more complex commandlines. see the blueprint plugin
+    // for an example.
+
+    return CR_OK;
+}
+
+/////////////////////////////////////////////////////
+// cycle logic
+//
+
+static void do_cycle(color_ostream &out) {
+    // mark that we have recently run
+    cycle_timestamp = world->frame_counter;
+
+    DEBUG(cycle,out).print("running %s cycle\n", plugin_name);
+
+    // TODO: logic that runs every get_config_val(CONFIG_CYCLE_TICKS) ticks
+}

plugins/examples/simple_command_example.cpp

@@ -0,0 +1,41 @@
+// This template is appropriate for plugins that simply provide one or more
+// commands, but don't need to be "enabled" to function.
+
+#include <string>
+#include <vector>
+
+#include "Debug.h"
+#include "PluginManager.h"
+
+using std::string;
+using std::vector;
+
+using namespace DFHack;
+
+DFHACK_PLUGIN("simple_command_example");
+
+namespace DFHack {
+    DBG_DECLARE(simple_command_example, log);
+}
+
+static command_result do_command(color_ostream &out, vector<string> &parameters);
+
+DFhackCExport command_result plugin_init(color_ostream &out, std::vector <PluginCommand> &commands) {
+    DEBUG(log,out).print("initializing %s\n", plugin_name);
+
+    commands.push_back(PluginCommand(
+        plugin_name,
+        "Short (~54 character) description of command.",
+        do_command));
+
+    return CR_OK;
+}
+
+static command_result do_command(color_ostream &out, vector<string> &parameters) {
+    // be sure to suspend the core if any DF state is read or modified
+    CoreSuspender suspend;
+
+    // TODO: command logic
+
+    return CR_OK;
+}

plugins/examples/skeleton.cpp

@@ -0,0 +1,193 @@
+// This is an example plugin that documents and implements all the plugin
+// callbacks and features. You can include it in the regular build by setting
+// the BUILD_SKELETON option in CMake to ON. Play with loading and unloading
+// the plugin in various game states (e.g. with and without a world loaded),
+// and see the debug messages get printed to the console.
+//
+// See the other example plugins in this directory for plugins that are
+// configured for specific use cases (but don't come with as many comments as
+// this one does).
+
+#include <string>
+#include <vector>
+
+#include "df/world.h"
+
+#include "Core.h"
+#include "Debug.h"
+#include "PluginManager.h"
+
+#include "modules/Persistence.h"
+#include "modules/World.h"
+
+using std::string;
+using std::vector;
+
+using namespace DFHack;
+
+// Expose the plugin name to the DFHack core, as well as metadata like the
+// DFHack version that this plugin was compiled with. This macro provides a
+// variable for the plugin name as const char * plugin_name.
+// The name provided must correspond to the filename --
+// skeleton.plug.so, skeleton.plug.dylib, or skeleton.plug.dll in this case
+DFHACK_PLUGIN("skeleton");
+
+// The identifier declared with this macro (i.e. is_enabled) is used to track
+// whether the plugin is in an "enabled" state. If you don't need enablement
+// for your plugin, you don't need this line. This variable will also be read
+// by the `plug` builtin command; when true the plugin will be shown as enabled.
+DFHACK_PLUGIN_IS_ENABLED(is_enabled);
+
+// Any globals a plugin requires (e.g. world) should be listed here.
+// For example, this line expands to "using df::global::world" and prevents the
+// plugin from being loaded if df::global::world is null (i.e. missing from
+// symbols.xml).
+REQUIRE_GLOBAL(world);
+
+// logging levels can be dynamically controlled with the `debugfilter` command.
+// Actual plugins will likely want to set the default level to LINFO or LWARNING
+// instead of the LDEBUG used here.
+namespace DFHack {
+    // for configuration-related logging
+    DBG_DECLARE(skeleton, status, DebugCategory::LDEBUG);
+    // for plugin_onupdate logging
+    DBG_DECLARE(skeleton, onupdate, DebugCategory::LDEBUG);
+    // for command-related logging
+    DBG_DECLARE(skeleton, command, DebugCategory::LDEBUG);
+}
+
+static command_result command_callback1(color_ostream &out, vector<string> &parameters);
+
+// run when the plugin is loaded
+DFhackCExport command_result plugin_init(color_ostream &out, std::vector<PluginCommand> &commands) {
+    DEBUG(status,out).print("initializing %s\n", plugin_name);
+
+    // For in-tree plugins, don't use the "usage" parameter of PluginCommand.
+    // Instead, add an .rst file with the same name as the plugin to the
+    // docs/plugins/ directory.
+    commands.push_back(PluginCommand(
+        "skeleton",
+        "Short (~54 character) description of command.", // to use one line in the ``[DFHack]# ls`` output
+        command_callback1));
+    return CR_OK;
+}
+
+// run when the plugin is unloaded
+DFhackCExport command_result plugin_shutdown(color_ostream &out) {
+    DEBUG(status,out).print("shutting down %s\n", plugin_name);
+
+    // You *MUST* kill all threads you created before this returns.
+    // If everything fails, just return CR_FAILURE. Your plugin will be
+    // in a zombie state, but things won't crash.
+    return CR_OK;
+
+}
+
+// run when the `enable` or `disable` command is run with this plugin name as
+// an argument
+DFhackCExport command_result plugin_enable(color_ostream &out, bool enable) {
+    DEBUG(status,out).print("%s from the API\n", enable ? "enabled" : "disabled");
+
+    // you have to maintain the state of the is_enabled variable yourself. it
+    // doesn't happen automatically.
+    is_enabled = enable;
+    return CR_OK;
+}
+
+// Called to notify the plugin about important state changes.
+// Invoked with DF suspended, and always before the matching plugin_onupdate.
+// More event codes may be added in the future.
+DFhackCExport command_result plugin_onstatechange(color_ostream &out, state_change_event event) {
+    switch (event) {
+        case SC_UNKNOWN:
+            DEBUG(status,out).print("game state changed: SC_UNKNOWN\n");
+            break;
+        case SC_WORLD_LOADED:
+            DEBUG(status,out).print("game state changed: SC_WORLD_LOADED\n");
+            break;
+        case SC_WORLD_UNLOADED:
+            DEBUG(status,out).print("game state changed: SC_WORLD_UNLOADED\n");
+            break;
+        case SC_MAP_LOADED:
+            DEBUG(status,out).print("game state changed: SC_MAP_LOADED\n");
+            break;
+        case SC_MAP_UNLOADED:
+            DEBUG(status,out).print("game state changed: SC_MAP_UNLOADED\n");
+            break;
+        case SC_VIEWSCREEN_CHANGED:
+            DEBUG(status,out).print("game state changed: SC_VIEWSCREEN_CHANGED\n");
+            break;
+        case SC_CORE_INITIALIZED:
+            DEBUG(status,out).print("game state changed: SC_CORE_INITIALIZED\n");
+            break;
+        case SC_BEGIN_UNLOAD:
+            DEBUG(status,out).print("game state changed: SC_BEGIN_UNLOAD\n");
+            break;
+        case SC_PAUSED:
+            DEBUG(status,out).print("game state changed: SC_PAUSED\n");
+            break;
+        case SC_UNPAUSED:
+            DEBUG(status,out).print("game state changed: SC_UNPAUSED\n");
+            break;
+    }
+
+    return CR_OK;
+}
+
+// Whatever you put here will be done in each game frame refresh. Don't abuse it.
+// Note that if the plugin implements the enabled API, this function is only called
+// if the plugin is enabled.
+DFhackCExport command_result plugin_onupdate (color_ostream &out) {
+    DEBUG(onupdate,out).print(
+        "onupdate called (run 'debugfilter set info skeleton onupdate' to stop"
+        " seeing these messages)\n");
+
+    return CR_OK;
+}
+
+// If you need to save or load world-specific data, define these functions.
+// plugin_save_data is called when the game might be about to save the world,
+// and plugin_load_data is called whenever a new world is loaded. If the plugin
+// is loaded or unloaded while a world is active, plugin_save_data or
+// plugin_load_data will be called immediately.
+DFhackCExport command_result plugin_save_data (color_ostream &out) {
+    DEBUG(status,out).print("save or unload is imminent; time to persist state\n");
+
+    // Call functions in the Persistence module here. If your PersistantDataItem
+    // objects are already up to date, then they will get persisted with the
+    // save automatically and there is nothing extra you need to do here.
+    return CR_OK;
+}
+
+DFhackCExport command_result plugin_load_data (color_ostream &out) {
+    DEBUG(status,out).print("world is loading; time to load persisted state\n");
+
+    // Call functions in the Persistence module here. See
+    // persistent_per_save_example.cpp for an example.
+    return CR_OK;
+}
+
+// This is the callback we registered in plugin_init. Note that while plugin
+// callbacks are called with the core suspended, command callbacks are called
+// from a different thread and need to explicity suspend the core if they
+// interact with Lua or DF game state (most commands do at least one of these).
+static command_result command_callback1(color_ostream &out, vector<string> &parameters) {
+    DEBUG(command,out).print("%s command called with %zu parameters\n",
+        plugin_name, parameters.size());
+
+    // I'll say it again: always suspend the core in command callbacks unless
+    // all your data is local.
+    CoreSuspender suspend;
+
+    // Return CR_WRONG_USAGE to print out your help text. The help text is
+    // sourced from the associated rst file in docs/plugins/. The same help will
+    // also be returned by 'help your-command'.
+
+    // simple commandline parsing can be done in C++, but there are lua libraries
+    // that can easily handle more complex commandlines. see the blueprint plugin
+    // for an example.
+
+    // TODO: do something according to the flags set in the options struct
+
+    return CR_OK;
+}

plugins/examples/ui_addition_example.cpp

@@ -0,0 +1,57 @@
+// This template is appropriate for plugins that can be enabled to make some
+// specific persistent change to the game, but don't need a world to be loaded
+// before they are enabled. These types of plugins typically register some sort
+// of hook on enable and clear the hook on disable. They are generally enabled
+// from dfhack.init and do not need to persist and reload their enabled state.
+
+#include <string>
+#include <vector>
+
+#include "df/viewscreen_titlest.h"
+
+#include "Debug.h"
+#include "PluginManager.h"
+#include "VTableInterpose.h"
+
+using std::string;
+using std::vector;
+
+using namespace DFHack;
+
+DFHACK_PLUGIN("ui_addition_example");
+DFHACK_PLUGIN_IS_ENABLED(is_enabled);
+
+namespace DFHack {
+    DBG_DECLARE(ui_addition_example, log);
+}
+
+// example of hooking a screen so the plugin code will run whenever the screen
+// is visible
+struct title_version_hook : df::viewscreen_titlest {
+    typedef df::viewscreen_titlest interpose_base;
+
+    DEFINE_VMETHOD_INTERPOSE(void, render, ()) {
+        INTERPOSE_NEXT(render)();
+
+        // TODO: injected render logic here
+    }
+};
+IMPLEMENT_VMETHOD_INTERPOSE(title_version_hook, render);
+
+DFhackCExport command_result plugin_shutdown (color_ostream &out) {
+    DEBUG(log,out).print("shutting down %s\n", plugin_name);
+    INTERPOSE_HOOK(title_version_hook, render).remove();
+    return CR_OK;
+}
+
+DFhackCExport command_result plugin_enable (color_ostream &out, bool enable) {
+    if (enable != is_enabled) {
+        DEBUG(log,out).print("%s %s\n", plugin_name,
+                             is_enabled ? "enabled" : "disabled");
+        if (!INTERPOSE_HOOK(title_version_hook, render).apply(enable))
+            return CR_FAILURE;
+
+        is_enabled = enable;
+    }
+    return CR_OK;
+}

plugins/hotkeys.cpp

@@ -44,6 +44,7 @@ static void find_active_keybindings(df::viewscreen *screen)
     sorted_keys.clear();
 
     vector<string> valid_keys;
+
     for (char c = 'A'; c <= 'Z'; c++)
     {
         valid_keys.push_back(string(&c, 1));
@@ -54,6 +55,8 @@ static void find_active_keybindings(df::viewscreen *screen)
         valid_keys.push_back("F" + int_to_string(i));
     }
 
+    valid_keys.push_back("`");
+
     auto current_focus = Gui::getFocusString(screen);
     for (int shifted = 0; shifted < 2; shifted++)
     {

plugins/liquids.cpp

@@ -55,6 +55,7 @@ using namespace df::enums;
 DFHACK_PLUGIN("liquids");
 REQUIRE_GLOBAL(world);
 
+static const char * HISTORY_FILE = "dfhack-config/liquids.history";
 CommandHistory liquids_hist;
 
 command_result df_liquids (color_ostream &out, vector <string> & parameters);
@@ -62,7 +63,7 @@ command_result df_liquids_here (color_ostream &out, vector <string> & parameters
 
 DFhackCExport command_result plugin_init ( color_ostream &out, std::vector <PluginCommand> &commands)
 {
-    liquids_hist.load("liquids.history");
+    liquids_hist.load(HISTORY_FILE);
     commands.push_back(PluginCommand(
         "liquids", "Place magma, water or obsidian.",
         df_liquids, true,
@@ -80,7 +81,7 @@ DFhackCExport command_result plugin_init ( color_ostream &out, std::vector <Plug
 
 DFhackCExport command_result plugin_shutdown ( color_ostream &out )
 {
-    liquids_hist.save("liquids.history");
+    liquids_hist.save(HISTORY_FILE);
     return CR_OK;
 }
 

plugins/listcolumn.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #include "uicommon.h"
 
 using df::global::enabler;

plugins/lua/autobutcher.lua

@@ -0,0 +1,80 @@
+local _ENV = mkmodule('plugins.autobutcher')
+
+local argparse = require('argparse')
+
+local function is_int(val)
+    return val and val == math.floor(val)
+end
+
+local function is_positive_int(val)
+    return is_int(val) and val > 0
+end
+
+local function check_nonnegative_int(str)
+    local val = tonumber(str)
+    if is_positive_int(val) or val == 0 then return val end
+    qerror('expecting a non-negative integer, but got: '..tostring(str))
+end
+
+local function process_args(opts, args)
+    if args[1] == 'help' then
+        opts.help = true
+        return
+    end
+
+    return argparse.processArgsGetopt(args, {
+            {'h', 'help', handler=function() opts.help = true end},
+        })
+end
+
+local function process_races(opts, races, start_idx)
+    if #races < start_idx then
+        qerror('missing list of races (or "all" or "new" keywords)')
+    end
+    for i=start_idx,#races do
+        local race = races[i]
+        if race == 'all' then
+            opts.races_all = true
+        elseif race == 'new' then
+            opts.races_new = true
+        else
+            local str = df.new('string')
+            str.value = race
+            opts.races:insert('#', str)
+        end
+    end
+end
+
+function parse_commandline(opts, ...)
+    local positionals = process_args(opts, {...})
+
+    local command = positionals[1]
+    if command then opts.command = command end
+
+    if opts.help or not command or command == 'now' or
+            command == 'autowatch' or command == 'noautowatch' or
+            command == 'list' or command == 'list_export' then
+        return
+    end
+
+    if command == 'watch' or command == 'unwatch' or command == 'forget' then
+        process_races(opts, positionals, 2)
+    elseif command == 'target' then
+        opts.fk = check_nonnegative_int(positionals[2])
+        opts.mk = check_nonnegative_int(positionals[3])
+        opts.fa = check_nonnegative_int(positionals[4])
+        opts.ma = check_nonnegative_int(positionals[5])
+        process_races(opts, positionals, 6)
+    elseif command == 'ticks' then
+        local ticks = tonumber(positionals[2])
+        if not is_positive_int(arg) then
+            qerror('number of ticks must be a positive integer: ' .. ticks)
+        else
+            opts.ticks = ticks
+        end
+    else
+        qerror(('unrecognized command: "%s"'):format(command))
+    end
+end
+
+return _ENV