Merge pull request #2297 from myk002/myk_count_doku

Generate tag indices

conf.py

@@ -58,9 +58,6 @@ def doc_dir(dirname, files, prefix):
 def doc_all_dirs():
     """Collect the commands and paths to include in our docs."""
     tools = []
-    # TODO: as we scan the docs, parse out the tags and short descriptions and
-    # build a map for use in generating the tags pages and links in the tool
-    # doc footers
     for root, _, files in os.walk('docs/builtins'):
         tools.extend(doc_dir(root, files, os.path.relpath(root, 'docs/builtins')))
     for root, _, files in os.walk('docs/plugins'):
@@ -69,59 +66,16 @@ def doc_all_dirs():
         tools.extend(doc_dir(root, files, os.path.relpath(root, 'scripts/docs')))
     return tuple(tools)
 
-DOC_ALL_DIRS = doc_all_dirs()
-
-
-def get_tags():
-    groups = {}
-    group_re = re.compile(r'"([^"]+)"')
-    tag_re = re.compile(r'- `tag/([^`]+)`: (.*)')
-    with open('docs/Tags.rst') as f:
-        lines = f.readlines()
-        for line in lines:
-            line = line.strip()
-            m = re.match(group_re, line)
-            if m:
-                group = m.group(1)
-                groups[group] = []
-                continue
-            m = re.match(tag_re, line)
-            if m:
-                tag = m.group(1)
-                desc = m.group(2)
-                groups[group].append((tag, desc))
-    return groups
-
-
-def generate_tag_indices():
-    os.makedirs('docs/tags', mode=0o755, exist_ok=True)
-    tag_groups = get_tags()
-    for tag_group in tag_groups:
-        with write_file_if_changed(('docs/tags/by{group}.rst').format(group=tag_group)) as topidx:
-            for tag_tuple in tag_groups[tag_group]:
-                tag = tag_tuple[0]
-                with write_file_if_changed(('docs/tags/{name}.rst').format(name=tag)) as tagidx:
-                    tagidx.write('TODO: add links to the tools that have this tag')
-                topidx.write(('.. _tag/{name}:\n\n').format(name=tag))
-                topidx.write(('{name}\n').format(name=tag))
-                topidx.write(('{underline}\n').format(underline='*'*len(tag)))
-                topidx.write(('{desc}\n\n').format(desc=tag_tuple[1]))
-                topidx.write(('.. include:: /docs/tags/{name}.rst\n\n').format(name=tag))
-
 
 def write_tool_docs():
     """
     Creates a file for each tool with the ".. include::" directives to pull in
-    the original documentation. Then we generate a label and useful info in the
-    footer.
+    the original documentation.
     """
-    for k in DOC_ALL_DIRS:
+    for k in doc_all_dirs():
         header = ':orphan:\n'
         label = ('.. _{name}:\n\n').format(name=k[0])
         include = ('.. include:: /{path}\n\n').format(path=k[1])
-        # TODO: generate a footer with links to tools that share at least one
-        # tag with this tool. Just the tool names, strung across the bottom of
-        # the page in one long wrapped line, similar to how the wiki does it
         os.makedirs(os.path.join('docs/tools', os.path.dirname(k[0])),
                     mode=0o755, exist_ok=True)
         with write_file_if_changed('docs/tools/{}.rst'.format(k[0])) as outfile:
@@ -131,9 +85,8 @@ def write_tool_docs():
             outfile.write(include)
 
 
-# Actually call the docs generator and run test
 write_tool_docs()
-generate_tag_indices()
+
 
 # -- General configuration ------------------------------------------------
 
@@ -315,11 +268,9 @@ html_sidebars = {
     ]
 }
 
-# If false, no module index is generated.
-html_domain_indices = False
-
-# If false, no genindex.html is generated.
-html_use_index = True
+# generate domain indices but not the (unused) genindex
+html_use_index = False
+html_domain_indices = True
 
 # don't link to rst sources in the generated pages
 html_show_sourcelink = False

docs/Core.rst

@@ -185,7 +185,7 @@ where ``*`` can be any string, including the empty string.
 A world being loaded can mean a fortress, an adventurer, or legends mode.
 
 These files are best used for non-persistent commands, such as setting
-a `tag/bugfix` script to run on `repeat`.
+a `bugfix-tag-index` script to run on `repeat`.
 
 
 .. _onMapLoad.init:

docs/Memory-research.rst

@@ -63,7 +63,7 @@ are not built by default, but can be built by setting the ``BUILD_DEVEL``
 Scripts
 ~~~~~~~
 
-Several `development tools <tag/dev>` can be useful for memory research.
+Several `development tools <dev-tag-index>` can be useful for memory research.
 These include (but are not limited to):
 
 - `devel/dump-offsets`

docs/Tags.rst

@@ -13,36 +13,36 @@ for the tag assignment spreadsheet.
 
 "when" tags
 -----------
-- `tag/adventure`: Tools that are useful while in adventure mode. Note that some tools only tagged with "fort" might also work in adventure mode, but not always in expected ways. Feel free to experiment, though!
-- `tag/dfhack`: Tools that you use to run DFHack commands or interact with the DFHack library. This tag also includes tools that help you manage the DF game itself (e.g. settings, saving, etc.)
-- `tag/embark`: Tools that are useful while on the fort embark screen or while creating an adventurer.
-- `tag/fort`: Tools that are useful while in fort mode.
-- `tag/legends`: Tools that are useful while in legends mode.
+- `adventure <adventure-tag-index>`: Tools that are useful while in adventure mode. Note that some tools only tagged with "fort" might also work in adventure mode, but not always in expected ways. Feel free to experiment, though!
+- `dfhack <dfhack-tag-index>`: Tools that you use to run DFHack commands or interact with the DFHack library. This tag also includes tools that help you manage the DF game itself (e.g. settings, saving, etc.)
+- `embark <embark-tag-index>`: Tools that are useful while on the fort embark screen or while creating an adventurer.
+- `fort <fort-tag-index>`: Tools that are useful while in fort mode.
+- `legends <legends-tag-index>`: Tools that are useful while in legends mode.
 
 "why" tags
 ----------
-- `tag/armok`: Tools that give you complete control over an aspect of the game or provide access to information that the game intentionally keeps hidden.
-- `tag/auto`: Tools that run in the background and automatically manage routine, toilsome aspects of your fortress.
-- `tag/bugfix`: Tools that fix specific bugs, either permanently or on-demand.
-- `tag/design`: Tools that help you design your fort.
-- `tag/dev`: Tools that are useful when developing scripts or mods.
-- `tag/fps`: Tools that help you manage FPS drop.
-- `tag/gameplay`: Tools that introduce new gameplay elements.
-- `tag/inspection`: Tools that let you view information that is otherwise difficult to find.
-- `tag/productivity`: Tools that help you do things that you could do manually, but using the tool is better and faster.
+- `armok <armok-tag-index>`: Tools that give you complete control over an aspect of the game or provide access to information that the game intentionally keeps hidden.
+- `auto <auto-tag-index>`: Tools that run in the background and automatically manage routine, toilsome aspects of your fortress.
+- `bugfix <bugfix-tag-index>`: Tools that fix specific bugs, either permanently or on-demand.
+- `design <design-tag-index>`: Tools that help you design your fort.
+- `dev <dev-tag-index>`: Tools that are useful when developing scripts or mods.
+- `fps <fps-tag-index>`: Tools that help you manage FPS drop.
+- `gameplay <gameplay-tag-index>`: Tools that introduce new gameplay elements.
+- `inspection <inspection-tag-index>`: Tools that let you view information that is otherwise difficult to find.
+- `productivity <productivity-tag-index>`: Tools that help you do things that you could do manually, but using the tool is better and faster.
 
 "what" tags
 -----------
-- `tag/animals`: Tools that interact with animals.
-- `tag/buildings`: Tools that interact with buildings and furniture.
-- `tag/graphics`: Tools that interact with game graphics.
-- `tag/interface`: Tools that interact with or extend the DF user interface.
-- `tag/items`: Tools that interact with in-game items.
-- `tag/jobs`: Tools that interact with jobs.
-- `tag/labors`: Tools that deal with labor assignment.
-- `tag/map`: Tools  that interact with the game map.
-- `tag/military`: Tools that interact with the military.
-- `tag/plants`: Tools that interact with trees, shrubs, and crops.
-- `tag/stockpiles`: Tools that interact wtih stockpiles.
-- `tag/units`: Tools that interact with units.
-- `tag/workorders`: Tools that interact with workorders.
+- `animals <animals-tag-index>`: Tools that interact with animals.
+- `buildings <buildings-tag-index>`: Tools that interact with buildings and furniture.
+- `graphics <graphics-tag-index>`: Tools that interact with game graphics.
+- `interface <interface-tag-index>`: Tools that interact with or extend the DF user interface.
+- `items <items-tag-index>`: Tools that interact with in-game items.
+- `jobs <jobs-tag-index>`: Tools that interact with jobs.
+- `labors <labors-tag-index>`: Tools that deal with labor assignment.
+- `map <map-tag-index>`: Tools  that interact with the game map.
+- `military <military-tag-index>`: Tools that interact with the military.
+- `plants <plants-tag-index>`: Tools that interact with trees, shrubs, and crops.
+- `stockpiles <stockpiles-tag-index>`: Tools that interact wtih stockpiles.
+- `units <units-tag-index>`: Tools that interact with units.
+- `workorders <workorders-tag-index>`: Tools that interact with workorders.

docs/Tools.rst

@@ -34,7 +34,7 @@ DFHack tools are tagged with categories to make them easier to find. These
 categories are listed in the next few sections. Note that a tool can belong to
 more than one category. If you already know what you're looking for, try the
 `search` or Ctrl-F on this page. If you'd like to see the full list of tools in
-one flat list, please refer to the `alphabetized index <genindex>`.
+one flat list, please refer to the `annotated index <all-tag-index>`.
 
 DFHack tools by game mode
 -------------------------
@@ -50,3 +50,14 @@ DFHack tools by what they affect
 --------------------------------
 
 .. include:: tags/bywhat.rst
+
+All DFHack tools alphabetically
+-------------------------------
+
+.. toctree::
+  :glob:
+  :maxdepth: 1
+  :titlesonly:
+
+  tools/*
+  tools/*/*

docs/plugins/reveal.rst

@@ -7,19 +7,19 @@ reveal
     :summary: Reveals the map.
     :tags: adventure fort armok inspection map
 
-.. dfhack-tool:: unreveal
+.. dfhack-command:: unreveal
    :summary: Hides previously hidden tiles again.
 
-.. dfhack-tool:: revforget
+.. dfhack-command:: revforget
    :summary: Discard records about what was visible before revealing the map.
 
-.. dfhack-tool:: revtoggle
+.. dfhack-command:: revtoggle
    :summary: Switch between reveal and unreveal.
 
-.. dfhack-tool:: revflood
+.. dfhack-command:: revflood
    :summary: Hide everything, then reveal tiles with a path to the cursor.
 
-.. dfhack-tool:: nopause
+.. dfhack-command:: nopause
    :summary: Disable pausing.
 
 This reveals all z-layers in fort mode. It also works in adventure mode, but any

docs/sphinx_extensions/dfhack/tool_docs.py

@@ -3,9 +3,11 @@
 #   https://www.sphinx-doc.org/en/master/development/tutorials/recipe.html
 #   https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html#rst-directives
 
+from collections import defaultdict
 import logging
 import os
-from typing import List, Optional, Type
+import re
+from typing import Dict, Iterable, List, Optional, Tuple, Type
 
 import docutils.nodes as nodes
 from docutils.nodes import Node
@@ -13,13 +15,13 @@ import docutils.parsers.rst.directives as rst_directives
 import sphinx
 import sphinx.addnodes as addnodes
 import sphinx.directives
+from sphinx.domains import Domain, Index, IndexEntry
 from sphinx.util.docutils import SphinxDirective
 from sphinx.util.nodes import process_index_entry
 
 import dfhack.util
 
 
-
 logger = sphinx.util.logging.getLogger(__name__)
 
 
@@ -50,18 +52,6 @@ def make_summary(builder: sphinx.builders.Builder, summary: str) -> nodes.paragr
     para += nodes.inline(text=summary)
     return para
 
-def make_index(directive: SphinxDirective, name: str, summary: str) -> List[Node]:
-    targetid = 'index-%s' % directive.env.new_serialno('index')
-    targetnode = nodes.target('', '', ids=[targetid])
-    directive.state.document.note_explicit_target(targetnode)
-    indexnode = addnodes.index()
-    indexnode['entries'] = []
-    indexnode['inline'] = False
-    directive.set_source_info(indexnode)
-    entry_text = 'single: {}; {}'.format(name, summary)
-    indexnode['entries'].extend(process_index_entry(entry_text, targetnode['ids'][0]))
-    return [indexnode, targetnode]
-
 _KEYBINDS = {}
 _KEYBINDS_RENDERED = set()  # commands whose keybindings have been rendered
 
@@ -120,20 +110,39 @@ def check_missing_keybinds():
         logger.warning('Undocumented keybindings for command: %s', missing_command)
 
 
+_anchor_pattern = re.compile(r'^\d+')
+
+def to_anchor(name: str) -> str:
+    name = name.lower()
+    name = name.replace('/', '-')
+    name = re.sub(_anchor_pattern, '', name)
+    return name
+
 class DFHackToolDirectiveBase(sphinx.directives.ObjectDescription):
     has_content = False
     required_arguments = 0
     optional_arguments = 1
 
+    def get_tool_name_from_docname(self):
+        parts = self.env.docname.split('/')
+        if 'tools' in parts:
+            return '/'.join(parts[parts.index('tools') + 1:])
+        else:
+            return parts[-1]
+
     def get_name_or_docname(self):
         if self.arguments:
             return self.arguments[0]
-        else:
-            parts = self.env.docname.split('/')
-            if 'tools' in parts:
-                return '/'.join(parts[parts.index('tools') + 1:])
-            else:
-                return parts[-1]
+        return self.get_tool_name_from_docname()
+
+    def add_index_entries(self, name) -> None:
+        docname = self.env.docname
+        anchor = to_anchor(self.get_tool_name_from_docname())
+        tags = self.env.domaindata['tag-repo']['doctags'][docname]
+        indexdata = (name, self.options.get('summary', ''), '', docname, anchor, 0)
+        self.env.domaindata['all']['objects'].append(indexdata)
+        for tag in tags:
+            self.env.domaindata[tag]['objects'].append(indexdata)
 
     @staticmethod
     def wrap_box(*children: List[nodes.Node]) -> nodes.Admonition:
@@ -160,13 +169,15 @@ class DFHackToolDirective(DFHackToolDirectiveBase):
 
     def render_content(self) -> List[nodes.Node]:
         tag_paragraph = self.make_labeled_paragraph('Tags')
-        for tag in self.options.get('tags', []):
+        tags = self.options.get('tags', [])
+        self.env.domaindata['tag-repo']['doctags'][self.env.docname] = tags
+        for tag in tags:
             tag_paragraph += [
                 addnodes.pending_xref(tag, nodes.inline(text=tag), **{
                     'reftype': 'ref',
                     'refdomain': 'std',
-                    'reftarget': 'tag/' + tag,
-                    'refexplicit': False,
+                    'reftarget': tag + '-tag-index',
+                    'refexplicit': True,
                     'refwarn': True,
                 }),
                 nodes.inline(text=' | '),
@@ -175,7 +186,7 @@ class DFHackToolDirective(DFHackToolDirectiveBase):
 
         ret_nodes = [tag_paragraph]
         if 'no-command' in self.options:
-            ret_nodes += make_index(self, self.get_name_or_docname() + ' (plugin)', self.options.get('summary', ''))
+            self.add_index_entries(self.get_name_or_docname() + ' (plugin)')
             ret_nodes += [make_summary(self.env.app.builder, self.options.get('summary', ''))]
         return ret_nodes
 
@@ -193,25 +204,121 @@ class DFHackCommandDirective(DFHackToolDirectiveBase):
 
     def render_content(self) -> List[nodes.Node]:
         command = self.get_name_or_docname()
-        ret_nodes = [self.make_labeled_paragraph('Command', command, content_class=nodes.literal)]
-        ret_nodes += make_index(self, command, self.options.get('summary', ''))
-        ret_nodes += [
+        self.add_index_entries(command)
+        return [
+            self.make_labeled_paragraph('Command', command, content_class=nodes.literal),
             make_summary(self.env.app.builder, self.options.get('summary', '')),
             *render_dfhack_keybind(command, builder=self.env.app.builder),
         ]
-        return ret_nodes
 
 
+class TagRepoDomain(Domain):
+    name = 'tag-repo'
+    label = 'Holds tag associations per document'
+    initial_data = {'doctags': {}}
+
+    def merge_domaindata(self, docnames: List[str], otherdata: Dict) -> None:
+        self.data['doctags'].update(otherdata['doctags'])
+
+
+def get_tags():
+    groups = {}
+    group_re = re.compile(r'"([^"]+)"')
+    tag_re = re.compile(r'- `([^ ]+) <[^>]+>`: (.*)')
+    with open('docs/Tags.rst') as f:
+        lines = f.readlines()
+        for line in lines:
+            line = line.strip()
+            m = re.match(group_re, line)
+            if m:
+                group = m.group(1)
+                groups[group] = []
+                continue
+            m = re.match(tag_re, line)
+            if m:
+                tag = m.group(1)
+                desc = m.group(2)
+                groups[group].append((tag, desc))
+    return groups
+
+
+def tag_domain_get_objects(self):
+    for obj in self.data['objects']:
+        yield(obj)
+
+def tag_domain_merge_domaindata(self, docnames: List[str], otherdata: Dict) -> None:
+    seen = set()
+    objs = self.data['objects']
+    for obj in objs:
+        seen.add(obj[0])
+    for obj in otherdata['objects']:
+        if obj[0] not in seen:
+            objs.append(obj)
+    objs.sort()
+
+def tag_index_generate(self, docnames: Optional[Iterable[str]] = None) -> Tuple[List[Tuple[str, List[IndexEntry]]], bool]:
+    content = defaultdict(list)
+    for name, desc, _, docname, anchor, _ in self.domain.data['objects']:
+        first_letter = name[0].lower()
+        extra, descr = desc, ''
+        if self.domain.env.app.builder.format == 'html':
+            extra, descr = '', desc
+        content[first_letter].append(
+            IndexEntry(name, 0, docname, anchor, extra, '', descr))
+    return (sorted(content.items()), False)
+
+def register_index(app, tag, title):
+    domain_class = type(tag+'Domain', (Domain, ), {
+            'name': tag,
+            'label': 'Container domain for tag: ' + tag,
+            'initial_data': {'objects': []},
+            'merge_domaindata': tag_domain_merge_domaindata,
+            'get_objects': tag_domain_get_objects,
+        })
+    index_class = type(tag+'Index', (Index, ), {
+            'name': 'tag-index',
+            'localname': title,
+            'shortname': tag,
+            'generate': tag_index_generate,
+        })
+    app.add_domain(domain_class)
+    app.add_index_to_domain(tag, index_class)
+
+def init_tag_indices(app):
+    os.makedirs('docs/tags', mode=0o755, exist_ok=True)
+    tag_groups = get_tags()
+    for tag_group in tag_groups:
+        with dfhack.util.write_file_if_changed(('docs/tags/by{group}.rst').format(group=tag_group)) as topidx:
+            for tag_tuple in tag_groups[tag_group]:
+                tag, desc = tag_tuple[0], tag_tuple[1]
+                topidx.write(('- `{name} <{name}-tag-index>`\n').format(name=tag))
+                topidx.write(('    {desc}\n').format(desc=desc))
+                register_index(app, tag, desc)
+
+
+def update_index_titles(app):
+    for domain in app.env.domains.values():
+        for index in domain.indices:
+            if index.shortname == 'all':
+                continue
+            if app.builder.format == 'html':
+                index.localname = '"%s" tag index<h4>%s</h4>' % (index.shortname, index.localname)
+            else:
+                index.localname = '"%s" tag index - %s' % (index.shortname, index.localname)
+
 def register(app):
     app.add_directive('dfhack-tool', DFHackToolDirective)
     app.add_directive('dfhack-command', DFHackCommandDirective)
-
+    update_index_titles(app)
     _KEYBINDS.update(scan_all_keybinds(os.path.join(dfhack.util.DFHACK_ROOT, 'data', 'init')))
 
-
 def setup(app):
     app.connect('builder-inited', register)
 
+    app.add_domain(TagRepoDomain)
+    register_index(app, 'all', 'Index of DFHack tools')
+    init_tag_indices(app)
+
     # TODO: re-enable once detection is corrected
     # app.connect('build-finished', lambda *_: check_missing_keybinds())
 

library/lua/helpdb.lua

@@ -396,7 +396,7 @@ local function initialize_tags()
             desc = desc .. ' ' .. line
             tag_index[tag].description = desc
         else
-            _,_,tag,desc = line:find('^%* (%w+): (.+)')
+            _,_,tag,desc = line:find('^%* (%w+)[^:]*: (.+)')
             if not tag then goto continue end
             tag_index[tag] = {description=desc}
             in_desc = true