prepare for plugin doc split

.gitignore

@@ -15,10 +15,11 @@ build/VC2010
 !build/
 
 # Sphinx generated documentation
-docs/_*
+docs/changelogs/
 docs/html/
 docs/pdf/
 docs/text/
+docs/tools/
 
 # in-place build
 build/Makefile

CMakeLists.txt

@@ -505,7 +505,7 @@ if(BUILD_DOCS)
         DESTINATION ${DFHACK_USERDOC_DESTINATION}/docs)
     install(DIRECTORY ${dfhack_SOURCE_DIR}/docs/text/
         DESTINATION ${DFHACK_USERDOC_DESTINATION}/docs)
-    install(FILES docs/_auto/news.rst docs/_auto/news-dev.rst DESTINATION ${DFHACK_USERDOC_DESTINATION})
+    install(FILES docs/changelogs/news.rst docs/changelogs/news-dev.rst DESTINATION ${DFHACK_USERDOC_DESTINATION})
     install(FILES "README.html" DESTINATION "${DFHACK_DATA_DESTINATION}")
 endif()
 

conf.py

@@ -29,8 +29,6 @@ import sys
 from docutils import nodes
 from docutils.parsers.rst import roles
 
-sphinx_major_version = sphinx.version_info[0]
-
 def get_keybinds(root, files, keybindings):
     """Add keybindings in the specified files to the
     given keybindings dict.
@@ -85,107 +83,86 @@ def dfhack_keybind_role_func(role, rawtext, text, lineno, inliner,
 
 roles.register_canonical_role('dfhack-keybind', dfhack_keybind_role_func)
 
-# -- Autodoc for DFhack scripts -------------------------------------------
+# -- Autodoc for DFhack plugins and scripts -------------------------------
 
-def doc_dir(dirname, files):
+def doc_dir(dirname, files, prefix):
-    """Yield (command, includepath) for each script in the directory."""
+    """Yield (name, includepath) for each file in the directory."""
     sdir = os.path.relpath(dirname, '.').replace('\\', '/').replace('../', '')
+    if prefix == '.':
+        prefix = ''
+    else:
+        prefix += '/'
     for f in files:
-        if f[-3:] not in ('lua', '.rb'):
+        if f[-4:] != '.rst':
             continue
-        with open(os.path.join(dirname, f), 'r', encoding='utf8') as fstream:
+        yield prefix + f[:-4], sdir + '/' + f
-            text = [l.rstrip() for l in fstream.readlines() if l.strip()]
-        # Some legacy lua files use the ruby tokens (in 3rdparty scripts)
-        tokens = ('=begin', '=end')
-        if f[-4:] == '.lua' and any('[====[' in line for line in text):
-            tokens = ('[====[', ']====]')
-        command = None
-        for line in text:
-            if command and line == len(line) * '=':
-                yield command, sdir + '/' + f, tokens[0], tokens[1]
-                break
-            command = line
 
 
 def doc_all_dirs():
     """Collect the commands and paths to include in our docs."""
-    scripts = []
+    tools = []
-    for root, _, files in os.walk('scripts'):
+    # TODO: as we scan the docs, parse out the tags and short descriptions and
-        scripts.extend(doc_dir(root, files))
+    # build a map for use in generating the tags pages and links in the tool
-    return tuple(scripts)
+    # doc footers
+    for root, _, files in os.walk('docs/plugins'):
+        tools.extend(doc_dir(root, files, os.path.relpath(root, 'docs/plugins')))
+    for root, _, files in os.walk('scripts/docs'):
+        tools.extend(doc_dir(root, files, os.path.relpath(root, 'scripts/docs')))
+    return tuple(tools)
 
 DOC_ALL_DIRS = doc_all_dirs()
 
+def generate_tag_indices():
+    #TODO: generate docs/tags/<tag>.rst with the tag description and links to the
+    # tools that have that tag
+    os.makedirs('docs/tags', mode=0o755, exist_ok=True)
 
-def document_scripts():
-    """Autodoc for files with the magic script documentation marker strings.
 
-    Returns a dict of script-kinds to lists of .rst include directives.
+def write_tool_docs():
     """
-    # Next we split by type and create include directives sorted by command
+    Creates a file for each tool with the ".. include::" directives to pull in
-    kinds = {'base': [], 'devel': [], 'fix': [], 'gui': [], 'modtools': []}
+    the original documentation. Then we generate a label and useful info in the
-    for s in DOC_ALL_DIRS:
+    footer.
-        k_fname = s[0].split('/', 1)
-        if len(k_fname) == 1:
-            kinds['base'].append(s)
-        else:
-            kinds[k_fname[0]].append(s)
-
-    def template(arg):
-        tmp = '.. _{}:\n\n.. include:: /{}\n' +\
-            '   :start-after: {}\n   :end-before: {}\n'
-        if arg[0] in KEYBINDS:
-            tmp += '\n:dfhack-keybind:`{}`\n'.format(arg[0])
-        return tmp.format(*arg)
-
-    return {key: '\n\n'.join(map(template, sorted(value)))
-            for key, value in kinds.items()}
-
-
-def write_script_docs():
-    """
-    Creates a file for eack kind of script (base/devel/fix/gui/modtools)
-    with all the ".. include::" directives to pull out docs between the
-    magic strings.
     """
-    kinds = document_scripts()
+    for k in DOC_ALL_DIRS:
-    head = {
+        label = ('.. _{name}:\n\n').format(name=k[0])
-        'base': 'Basic Scripts',
+        # TODO: can we autogenerate the :dfhack-keybind: line? it would go beneath
-        'devel': 'Development Scripts',
+        # the tool header, which is currently in the middle of the included file.
-        'fix': 'Bugfixing Scripts',
+        # should we remove those headers from the doc files and just generate them
-        'gui': 'GUI Scripts',
+        # here? That might be easier. But then where will the tags go? It would
-        'modtools': 'Scripts for Modders'}
+        # look better if they were above the keybinds, but then we'd be in the
-    for k in head:
+        # same situation.
-        title = ('.. _scripts-{k}:\n\n{l}\n{t}\n{l}\n\n'
+        include = ('.. include:: /{path}\n\n').format(path=k[1])
-                 '.. include:: /scripts/{a}about.txt\n\n'
+        # TODO: generate a footer with links to tools that share at least one
-                 '.. contents:: Contents\n'
+        # tag with this tool. Just the tool names, strung across the bottom of
-                 '  :local:\n\n').format(
+        # the page in one long wrapped line, similar to how the wiki does it
-                     k=k, t=head[k],
-                     l=len(head[k])*'#',
-                     a=('' if k == 'base' else k + '/')
-                     )
         mode = 'w' if sys.version_info.major > 2 else 'wb'
-        with open('docs/_auto/{}.rst'.format(k), mode) as outfile:
+        os.makedirs(os.path.join('docs/tools', os.path.dirname(k[0])),
-            outfile.write(title)
+                    mode=0o755, exist_ok=True)
-            outfile.write(kinds[k])
+        with open('docs/tools/{}.rst'.format(k[0]), mode) as outfile:
+            if k[0] != 'search':
+                outfile.write(label)
+            outfile.write(include)
 
 
 def all_keybinds_documented():
     """Check that all keybindings are documented with the :dfhack-keybind:
     directive somewhere."""
-    configured_binds = set(KEYBINDS)
+    undocumented_binds = set(KEYBINDS)
-    script_commands = set(i[0] for i in DOC_ALL_DIRS)
+    tools = set(i[0] for i in DOC_ALL_DIRS)
-    with open('./docs/Plugins.rst') as f:
+    for t in tools:
-        plugin_binds = set(re.findall(':dfhack-keybind:`(.*?)`', f.read()))
+        with open(('./docs/tools/{}.rst').format(t)) as f:
-    undocumented_binds = configured_binds - script_commands - plugin_binds
+            tool_binds = set(re.findall(':dfhack-keybind:`(.*?)`', f.read()))
+            undocumented_binds -= tool_binds
     if undocumented_binds:
         raise ValueError('The following DFHack commands have undocumented '
                          'keybindings: {}'.format(sorted(undocumented_binds)))
 
 
 # Actually call the docs generator and run test
-write_script_docs()
+write_tool_docs()
-all_keybinds_documented()
+generate_tag_indices()
+#all_keybinds_documented() # comment out while we're transitioning
 
 # -- General configuration ------------------------------------------------
 
@@ -203,6 +180,8 @@ extensions = [
     'dfhack.lexer',
 ]
 
+sphinx_major_version = sphinx.version_info[0]
+
 def get_caption_str(prefix=''):
     return prefix + (sphinx_major_version >= 5 and '%s' or '')
 
@@ -282,11 +261,11 @@ today_fmt = html_last_updated_fmt = '%Y-%m-%d'
 # directories to ignore when looking for source files.
 exclude_patterns = [
     'README.md',
-    'docs/html*',
-    'depends/*',
     'build*',
-    'docs/_auto/news*',
+    'depends/*',
-    'docs/_changelogs/',
+    'docs/html/*',
+    'docs/text/*',
+    'docs/plugins/*',
     'scripts/docs/*',
     ]
 

docs/Core.rst

@@ -15,19 +15,18 @@ DFHack commands can be implemented in any of three ways:
 
 :builtin:   commands are implemented by the core of DFHack. They manage
             other DFHack tools, interpret commands, and control basic
-            aspects of DF (force pause or quit). They are documented
+            aspects of DF (force pause or quit).
-            `here <built-in-commands>`.
 
 :plugins:   are stored in ``hack/plugins/`` and must be compiled with the
             same version of DFHack.  They are less flexible than scripts,
             but used for complex or ongoing tasks because they run faster.
-            Plugins included with DFHack are documented `here <plugins-index>`.
 
 :scripts:   are Ruby or Lua scripts stored in ``hack/scripts/``.
             Because they don't need to be compiled, scripts are
             more flexible about versions, and easier to distribute.
-            Most third-party DFHack addons are scripts. All scripts included
+            Most third-party DFHack addons are scripts.
-            with DFHack are documented `here <scripts-index>`.
+
+All tools distributed with DFHack are documented `here <tools-index>`.
 
 Using DFHack Commands
 =====================
@@ -186,7 +185,7 @@ where ``*`` can be any string, including the empty string.
 A world being loaded can mean a fortress, an adventurer, or legends mode.
 
 These files are best used for non-persistent commands, such as setting
-a `fix <scripts-fix>` script to run on `repeat`.
+a `fix <tools-index>` script to run on `repeat`.
 
 
 .. _onMapLoad.init:

docs/Dev-intro.rst

@@ -41,7 +41,7 @@ Installed plugins live in the ``hack/plugins`` folder of a DFHack installation,
 and the `load` family of commands can be used to load a recompiled plugin
 without restarting DF.
 
-See `plugins-index` for a list of all plugins included in DFHack.
+Run `plug` at the DFHack prompt for a list of all plugins included in DFHack.
 
 Scripts
 -------
@@ -51,8 +51,9 @@ is more complete and currently better-documented, however. Referring to existing
 scripts as well as the API documentation can be helpful when developing new
 scripts.
 
-`Scripts included in DFHack <scripts-index>` live in a separate `scripts repository <https://github.com/dfhack/scripts>`_.
+`Scripts included in DFHack <tools-index>` live in a separate
-This can be found in the ``scripts`` submodule if you have
+`scripts repository <https://github.com/dfhack/scripts>`_. This can be found in
+the ``scripts`` submodule if you have
 `cloned DFHack <compile-how-to-get-the-code>`, or the ``hack/scripts`` folder
 of an installed copy of DFHack.
 

docs/Documentation.rst

@@ -318,10 +318,10 @@ changelogs are combined as part of the changelog build process:
 * ``scripts/changelog.txt`` for changes made to scripts in the ``scripts`` repo
 * ``library/xml/changelog.txt`` for changes made in the ``df-structures`` repo
 
-Building the changelogs generates two files: ``docs/_auto/news.rst`` and
+Building the changelogs generates two files: ``docs/changelogs/news.rst`` and
-``docs/_auto/news-dev.rst``. These correspond to `changelog` and `dev-changelog`
+``docs/changelogs/news-dev.rst``. These correspond to `changelog` and
-and contain changes organized by stable and development DFHack releases,
+`dev-changelog` and contain changes organized by stable and development DFHack
-respectively. For example, an entry listed under "0.44.05-alpha1" in
+releases, respectively. For example, an entry listed under "0.44.05-alpha1" in
 changelog.txt will be listed under that version in the development changelog as
 well, but under "0.44.05-r1" in the stable changelog (assuming that is the
 closest stable release after 0.44.05-alpha1). An entry listed under a stable

docs/History.rst

@@ -1350,7 +1350,7 @@ Misc improvements
 - `createitem`: in adventure mode it now defaults to the controlled unit as maker.
 - `autotrade`: adds "(Un)mark All" options to both panes of trade screen.
 - `mousequery`: several usability improvements; show live overlay (in menu area) of what's on the tile under the mouse cursor.
-- `search`: workshop profile search added.
+- `search-plugin`: workshop profile search added.
 - `dwarfmonitor`: add screen to summarise preferences of fortress dwarfs.
 - `getplants`: add autochop function to automate woodcutting.
 - `stocks`: added more filtering and display options.
@@ -1510,7 +1510,7 @@ New binary patches
 New Plugins
 -----------
 - `fix-armory`: Together with a couple of binary patches and the `gui/assign-rack` script, this plugin makes weapon racks, armor stands, chests and cabinets in properly designated barracks be used again for storage of squad equipment.
-- `search`: Adds an incremental search function to the Stocks, Trading, Stockpile and Unit List screens.
+- `search-plugin`: Adds an incremental search function to the Stocks, Trading, Stockpile and Unit List screens.
 - `automaterial`: Makes building constructions (walls, floors, fortifications, etc) a little bit easier by saving you from having to trawl through long lists of materials each time you place one.
 - Dfusion: Reworked to make use of lua modules, now all the scripts can be used from other scripts.
 - Eventful: A collection of lua events, that will allow new ways to interact with df world.

docs/Introduction.rst

@@ -21,7 +21,7 @@ enhancements by default, and more can be enabled.  There are also many tools
 You can even add third-party scripts and plugins to do almost anything!
 
 For modders, DFHack makes many things possible.  Custom reactions, new
-interactions, magic creature abilities, and more can be set through `scripts-modtools`
+interactions, magic creature abilities, and more can be set through `DFHack tools <tools-index>`
 and custom raws.  Non-standard DFHack scripts and inits can be stored in the
 raw directory, making raws or saves fully self-contained for distribution -
 or for coexistence in a single DF install, even with incompatible components.

docs/Memory-research.rst

@@ -63,7 +63,7 @@ are not built by default, but can be built by setting the ``BUILD_DEVEL``
 Scripts
 ~~~~~~~
 
-Several `development scripts <scripts-devel>` can be useful for memory research.
+Several `development scripts <tools-index>` can be useful for memory research.
 These include (but are not limited to):
 
 - `devel/dump-offsets`

docs/NEWS-dev.rst

@@ -17,4 +17,4 @@ See `changelog` for a list of changes grouped by stable releases.
   :local:
   :depth: 1
 
-.. include:: /docs/_auto/news-dev.rst
+.. include:: /docs/changelogs/news-dev.rst

docs/NEWS.rst

@@ -17,7 +17,7 @@ See `dev-changelog` for a list of changes grouped by development releases.
   :local:
   :depth: 1
 
-.. include:: /docs/_auto/news.rst
+.. include:: /docs/changelogs/news.rst
 
 
 Older Changelogs

docs/Removed.rst

@@ -59,6 +59,14 @@ script instead. You can use your existing .csv files. Just move them to the
 ``blueprints`` folder in your DF installation, and instead of
 ``fortplan file.csv`` run ``quickfort run file.csv``.
 
+.. _gui/no-dfhack-init:
+
+gui/no-dfhack-init
+==================
+Tool that warned the user when the ``dfhack.init`` file did not exist. Now that
+``dfhack.init`` is autogenerated in ``dfhack-config/init``, this warning is no
+longer necessary.
+
 .. _warn-stuck-trees:
 
 warn-stuck-trees

docs/Scripts.rst

@@ -1,19 +0,0 @@
-.. _scripts-index:
-
-##############
-DFHack Scripts
-##############
-
-Lua or ruby scripts placed in the :file:`hack/scripts/` directory
-are considered for execution as if they were native DFHack commands.
-
-The following pages document all the scripts in the DFHack standard library.
-
-.. toctree::
-   :maxdepth: 2
-
-   /docs/_auto/base
-   /docs/_auto/devel
-   /docs/_auto/fix
-   /docs/_auto/gui
-   /docs/_auto/modtools

docs/Tools.rst

@@ -1,13 +0,0 @@
-.. _tools-index:
-
-############
-DFHack Tools
-############
-
-These are the DFHack commands you can run.
-
-.. toctree::
-   :titlesonly:
-   :glob:
-
-   /docs/tools/*

docs/_auto/.gitignore

@@ -1 +0,0 @@
-*.rst

docs/build-pdf.sh

@@ -11,13 +11,4 @@
 cd $(dirname "$0")
 cd ..
 
-sphinx=sphinx-build
+"${SPHINX:-sphinx-build}" -M latexpdf -d build/docs/pdf . docs/pdf -w build/docs/pdf/_sphinx-warnings.txt -j "${JOBS:-auto}" "$@"
-if [ -n "$SPHINX" ]; then
-    sphinx=$SPHINX
-fi
-
-if [ -z "$JOBS" ]; then
-    JOBS=2
-fi
-
-"$sphinx" -M latexpdf . ./docs/pdf -w ./docs/_sphinx-warnings.txt -j "$JOBS" "$@"

docs/build.sh

@@ -11,13 +11,5 @@
 cd $(dirname "$0")
 cd ..
 
-sphinx=sphinx-build
+"${SPHINX:-sphinx-build}" -b html -d build/docs/html . docs/html -w build/docs/html/_sphinx-warnings.txt -j "${JOBS:-auto}" "$@"
-if [ -n "$SPHINX" ]; then
+"${SPHINX:-sphinx-build}" -b text -d build/docs/text . docs/text -w build/docs/text/_sphinx-warnings.txt -j "${JOBS:-auto}" "$@"
-    sphinx=$SPHINX
-fi
-
-if [ -z "$JOBS" ]; then
-    JOBS=2
-fi
-
-"$sphinx" -a -b html . ./docs/html -w ./docs/_sphinx-warnings.txt -j "$JOBS" "$@"

docs/changelogs/.gitignore

@@ -1 +1,2 @@
 *.txt
+*.rst

docs/guides/examples-guide.rst

@@ -28,8 +28,8 @@ it is useful (and customizable) for any fort. It includes the following config:
 - Calls `ban-cooking` for items that have important alternate uses and should
   not be cooked. This configuration is only set when a fortress is first
   started, so later manual changes will not be overridden.
-- Automates calling of various fort maintenance and `scripts-fix`, like
+- Automates calling of various fort maintenance scripts, like `cleanowned` and
-  `cleanowned` and `fix/stuckdoors`.
+  `fix/stuckdoors`.
 - Keeps your manager orders intelligently ordered with `orders` ``sort`` so no
   orders block other orders from ever getting completed.
 - Periodically enqueues orders to shear and milk shearable and milkable pets.

docs/index-tools.rst

@@ -0,0 +1,24 @@
+.. _tools-index:
+
+============
+DFHack Tools
+============
+
+These pages contain information about the plugins, scripts, and built-in
+commands distributed with DFHack.
+
+.. note::
+    In order to avoid user confusion, as a matter of policy all GUI tools
+    display the word :guilabel:`DFHack` on the screen somewhere while active.
+
+    When that is not appropriate because they merely add keybinding hints to
+    existing DF screens, they deliberately use red instead of green for the key.
+
+.. toctree::
+   :titlesonly:
+   :glob:
+
+   /docs/Tags
+   /docs/Builtin
+   /docs/tools/*
+   /docs/tools/*/*

docs/sphinx_extensions/dfhack/changelog.py

@@ -238,8 +238,8 @@ def generate_changelog(all=False):
     consolidate_changelog(stable_entries)
     consolidate_changelog(dev_entries)
 
-    print_changelog(versions, stable_entries, os.path.join(DOCS_ROOT, '_auto/news.rst'))
+    print_changelog(versions, stable_entries, os.path.join(DOCS_ROOT, 'changelogs/news.rst'))
-    print_changelog(versions, dev_entries, os.path.join(DOCS_ROOT, '_auto/news-dev.rst'))
+    print_changelog(versions, dev_entries, os.path.join(DOCS_ROOT, 'changelogs/news-dev.rst'))
 
     if all:
         for version in versions:
@@ -251,10 +251,10 @@ def generate_changelog(all=False):
             else:
                 version_entries = {version: dev_entries[version]}
             print_changelog([version], version_entries,
-                os.path.join(DOCS_ROOT, '_changelogs/%s-github.txt' % version),
+                os.path.join(DOCS_ROOT, 'changelogs/%s-github.txt' % version),
                 replace=False)
             print_changelog([version], version_entries,
-                os.path.join(DOCS_ROOT, '_changelogs/%s-reddit.txt' % version),
+                os.path.join(DOCS_ROOT, 'changelogs/%s-reddit.txt' % version),
                 replace=False,
                 prefix='> ')
 
@@ -264,7 +264,7 @@ def cli_entrypoint():
     import argparse
     parser = argparse.ArgumentParser()
     parser.add_argument('-a', '--all', action='store_true',
-        help='Print changelogs for all versions to docs/_changelogs')
+        help='Print changelogs for all versions to docs/changelogs')
     parser.add_argument('-c', '--check', action='store_true',
         help='Check that all entries are printed')
     args = parser.parse_args()
@@ -272,9 +272,9 @@ def cli_entrypoint():
     entries = generate_changelog(all=args.all)
 
     if args.check:
-        with open(os.path.join(DOCS_ROOT, '_auto/news.rst')) as f:
+        with open(os.path.join(DOCS_ROOT, 'changelogs/news.rst')) as f:
             content_stable = f.read()
-        with open(os.path.join(DOCS_ROOT, '_auto/news-dev.rst')) as f:
+        with open(os.path.join(DOCS_ROOT, 'changelogs/news-dev.rst')) as f:
             content_dev = f.read()
         for entry in entries:
             for description in entry.children:

index.rst

@@ -30,11 +30,7 @@ User Manual
    /docs/Installing
    /docs/Support
    /docs/Core
-   /docs/Builtin
+   /docs/index-tools
-   /docs/Plugins
-   /docs/Scripts
-   /docs/Tags
-   /docs/Tools
    /docs/guides/index
    /docs/index-about
    /docs/index-dev