Improve handling of in-script documentation

Closes issue #823.  This allows for clean unification of html docs and
the in-terminal help text for scripts, including handling in core rather
than on a per-script basis (see issue #947).

Contributing.rst

@@ -148,29 +148,32 @@ as well as getting help, you'll be providing valuable feedback that
 makes it easier for future readers!
 
 Scripts can use a custom autodoc function, based on the Sphinx ``include``
-directive and Ruby docstring conventions - any lines between ``=begin`` and
+directive - anything between the tokens is copied into the appropriate scripts
-``=end`` are copied into the appropriate scripts documentation page.
+documentation page.  For Ruby, we follow the built-in docstring convention
-They **must** have a heading which exactly matches the command, underlined
+(``=begin`` and ``=end``).  For Lua, the tokens are ``[====[`` and ``]====]``
+- ordinary multi-line strings.  It is highly encouraged to reuse this string
+as the in-console documentation by (eg.) printing it when a ``-help`` argument
+is given.
+
+The docs **must** have a heading which exactly matches the command, underlined
 with ``=====`` to the same length.  For example, a lua file would have::
 
-    --[[=begin
+    local helpstr = [====[
 
     add-thought
     ===========
     Adds a thought or emotion to the selected unit.  Can be used by other scripts,
     or the gui invoked by running ``add-thought gui`` with a unit selected.
 
-    =end]]
+    ]====]
 
-Ruby scripts use the same syntax, but obviously omit the leading ``--[[`` and
-trailing ``]]`` which denote a multiline comment in lua.
-``=begin`` and ``=end`` are native syntax (and matched in lua for convenience).
 
 Where the heading for a section is also the name of a command, the spelling
 and case should exactly match the command to enter in the DFHack command line.
 
-Try to keep lines within 80-100 characters, so it's readable in plain text -
+Try to keep lines within 80-100 characters, so it's readable in plain text
-Sphinx (our documentation system) will make sure paragraphs flow.
+in the terminal - Sphinx (our documentation system) will make sure
+paragraphs flow.
 
 If there aren't many options or examples to show, they can go in a paragraph of
 text.  Use double-backticks to put commands in monospaced font, like this::
@@ -208,8 +211,7 @@ section like this::
     =========
 
 Add link targets if you need them, but otherwise plain headings are preferred.
-Scripts using the in-source docs option, which should be all of them, have
+Scripts have link targets created automatically.
-link targets created automatically.
 
 Other ways to help
 ==================

NEWS.rst

@@ -54,6 +54,7 @@ Misc Improvements
 - `catsplosion`: now a lua script instead of a plugin
 - `fix/diplomats`: replaces ``fixdiplomats``
 - `fix/merchants`: replaces ``fixmerchants``
+- Unified script documentation and in-terminal help options
 
 Removed
 -------

conf.py

@@ -1,21 +1,25 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-#
+"""
-# DFHack documentation build configuration file
+DFHack documentation build configuration file
-#
+
-# This file is execfile()d with the current directory set to its
+This file is execfile()d with the current directory set to its
-# containing dir.
+containing dir.
-#
+
-# Note that not all possible configuration values are present in this
+Note that not all possible configuration values are present in this
-# autogenerated file.
+autogenerated file.
-#
+
-# All configuration values have a default; values that are commented out
+All configuration values have a default; values that are commented out
-# serve to show the default.
+serve to show the default.
+"""
+
+# pylint:disable=redefined-builtin
 
 import fnmatch
 from io import open
+from itertools import starmap
 import os
-import shlex
+import shlex  # pylint:disable=unused-import
 import sys
 
 
@@ -29,10 +33,14 @@ def doc_dir(dirname, files):
             continue
         with open(os.path.join(dirname, f), 'r', encoding='utf8') as fstream:
             text = [l.rstrip() for l in fstream.readlines() if l.strip()]
+        # Some legacy lua files use the ruby tokens (in 3rdparty scripts)
+        tokens = ('=begin', '=end')
+        if f[-4:] == '.lua' and any('[====[' in line for line in text):
+            tokens = ('[====[', ']====]')
         command = None
         for line in text:
             if command and line == len(line) * '=':
-                yield command, sdir + '/' + f
+                yield command, sdir + '/' + f, tokens[0], tokens[1]
                 break
             command = line
 
@@ -40,9 +48,7 @@ def doc_dir(dirname, files):
 def document_scripts():
     """Autodoc for files with the magic script documentation marker strings.
 
-    Creates a file for eack kind of script (base/devel/fix/gui/modtools)
+    Returns a dict of script-kinds to lists of .rst include directives.
-    with all the ".. include::" directives to pull out docs between the
-    magic strings.
     """
     # First, we collect the commands and paths to include in our docs
     scripts = []
@@ -55,15 +61,19 @@ def document_scripts():
         if len(k_fname) == 1:
             kinds['base'].append(s)
         else:
-            kinds.get(k_fname[0], []).append(s)
+            kinds[k_fname[0]].append(s)
-    template = ('.. _{}:\n\n'
+    template = '.. _{}:\n\n.. include:: /{}\n' +\
-                '.. include:: /{}\n'
+        '   :start-after: {}\n   :end-before: {}\n'
-                '   :start-after: =begin\n'
+    return {key: '\n\n'.join(starmap(template.format, sorted(value)))
-                '   :end-before: =end\n')
+            for key, value in kinds.items()}
-    for key, value in kinds.items():
+
-        kinds[key] = [template.format(x[0], x[1])
+def write_script_docs():
-                      for x in sorted(value, key=lambda x: x[0])]
+    """
-    # Finally, we write our _auto/* files for each kind of script
+    Creates a file for eack kind of script (base/devel/fix/gui/modtools)
+    with all the ".. include::" directives to pull out docs between the
+    magic strings.
+    """
+    kinds = document_scripts()
     if not os.path.isdir('docs/_auto'):
         os.mkdir('docs/_auto')
     head = {
@@ -76,15 +86,18 @@ def document_scripts():
         title = ('.. _{k}:\n\n{l}\n{t}\n{l}\n\n'
                  '.. include:: /scripts/{a}about.txt\n\n'
                  '.. contents::\n\n').format(
-            k=k, t=head[k], l=len(head[k])*'#', a=('' if k=='base' else k+'/'))
+                     k=k, t=head[k],
+                     l=len(head[k])*'#',
+                     a=('' if k == 'base' else k + '/')
+                     )
         mode = 'w' if sys.version_info.major > 2 else 'wb'
         with open('docs/_auto/{}.rst'.format(k), mode) as outfile:
             outfile.write(title)
-            outfile.write('\n\n'.join(kinds[k]))
+            outfile.write(kinds[k])
 
 
 # Actually call the docs generator
-document_scripts()
+write_script_docs()
 
 
 # -- General configuration ------------------------------------------------
@@ -135,7 +148,7 @@ author = 'The DFHack Team'
 
 def get_version():
     """Return the DFHack version string, from CMakeLists.txt"""
-    version = release = ''
+    version = release = ''  #pylint:disable=redefined-outer-name
     try:
         with open('CMakeLists.txt') as f:
             for s in f.readlines():
@@ -157,11 +170,8 @@ version = release = get_version()
 # Usually you set "language" from the command line for these cases.
 language = None
 
-# There are two options for replacing |today|: either, you set today to some
+# strftime format for |today| and 'Last updated on:' timestamp at page bottom
-# non-false value, then it is used:
+today_fmt = html_last_updated_fmt = '%Y-%m-%d'
-#today = ''
-# Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -202,9 +212,6 @@ html_theme_options = {
     'travis_button': False,
 }
 
-# Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
-
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
 #html_title = None
@@ -226,19 +233,6 @@ html_favicon = 'docs/styles/dfhack-icon.ico'
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['docs/styles']
 
-# Add any extra paths that contain custom files (such as robots.txt or
-# .htaccess) here, relative to this directory. These files are copied
-# directly to the root of the documentation.
-#html_extra_path = []
-
-# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
-# using the given strftime format.
-html_last_updated_fmt = '%Y-%m-%d'
-
-# If true, SmartyPants will be used to convert quotes and dashes to
-# typographically correct entities.
-#html_use_smartypants = True
-
 # Custom sidebar templates, maps document names to template names.
 html_sidebars = {
     '**': [
@@ -249,52 +243,14 @@ html_sidebars = {
     ]
 }
 
-# Additional templates that should be rendered to pages, maps page names to
-# template names.
-#html_additional_pages = {}
-
 # If false, no module index is generated.
 html_domain_indices = False
 
 # If false, no index is generated.
 html_use_index = False
 
-# If true, the index is split into individual pages for each letter.
-#html_split_index = False
-
-# If true, links to the reST sources are added to the pages.
-#html_show_sourcelink = True
-
-# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-#html_show_sphinx = True
-
-# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#html_show_copyright = True
-
-# If true, an OpenSearch description file will be output, and all pages will
-# contain a <link> tag referring to it.  The value of this option must be the
-# base URL from which the finished HTML is served.
-#html_use_opensearch = ''
-
-# Output file base name for HTML help builder.
-htmlhelp_basename = 'DFHackdoc'
-
 # -- Options for LaTeX output ---------------------------------------------
 
-latex_elements = {
-# The paper size ('letterpaper' or 'a4paper').
-#'papersize': 'letterpaper',
-
-# The font size ('10pt', '11pt' or '12pt').
-#'pointsize': '10pt',
-
-# Additional stuff for the LaTeX preamble.
-#'preamble': '',
-
-# Latex figure (float) alignment
-#'figure_align': 'htbp',
-}
-
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
@@ -302,60 +258,3 @@ latex_documents = [
     (master_doc, 'DFHack.tex', 'DFHack Documentation',
      'The DFHack Team', 'manual'),
 ]
-
-# The name of an image file (relative to this directory) to place at the top of
-# the title page.
-#latex_logo = None
-
-# For "manual" documents, if this is true, then toplevel headings are parts,
-# not chapters.
-#latex_use_parts = False
-
-# If true, show page references after internal links.
-#latex_show_pagerefs = False
-
-# If true, show URL addresses after external links.
-#latex_show_urls = False
-
-# Documents to append as an appendix to all manuals.
-#latex_appendices = []
-
-# If false, no module index is generated.
-#latex_domain_indices = True
-
-
-# -- Options for manual page output ---------------------------------------
-
-# One entry per manual page. List of tuples
-# (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, 'dfhack', 'DFHack Documentation',
-     [author], 1)
-]
-
-# If true, show URL addresses after external links.
-#man_show_urls = False
-
-
-# -- Options for Texinfo output -------------------------------------------
-
-# Grouping the document tree into Texinfo files. List of tuples
-# (source start file, target name, title, author,
-#  dir menu entry, description, category)
-texinfo_documents = [
-  (master_doc, 'DFHack', 'DFHack Documentation',
-   author, 'DFHack', 'One line description of project.',
-   'Miscellaneous'),
-]
-
-# Documents to append as an appendix to all manuals.
-#texinfo_appendices = []
-
-# If false, no module index is generated.
-#texinfo_domain_indices = True
-
-# How to display URL addresses: 'footnote', 'no', or 'inline'.
-#texinfo_show_urls = 'footnote'
-
-# If true, do not generate a @detailmenu in the "Top" node's menu.
-#texinfo_no_detailmenu = False

docs/Authors.rst

@@ -95,6 +95,7 @@ Sebastian Wolfertz      Enkrod
 Seth Woodworth          sethwoodworth
 simon
 Simon Jackson           sizeak
+stolencatkarma
 sv-esk                  sv-esk
 Tacomagic
 Tim Walberg             twalberg

travis/script-in-readme.py

@@ -17,7 +17,7 @@ def check_ls(fname, line):
     """Check length & existence of leading comment for "ls" builtin command."""
     line = line.strip()
     comment = '--' if fname.endswith('.lua') else '#'
-    if line.endswith('=begin') or not line.startswith(comment):
+    if '[====[' in line or not line.startswith(comment):
         print('Error: no leading comment in ' + fname)
         return 1
     return 0
@@ -25,13 +25,15 @@ def check_ls(fname, line):
 
 def check_file(fname):
     errors, doclines = 0, []
+    tok1, tok2 = ('=begin', '=end') if fname.endswith('.rb') else \
+        ('[====[', ']====]')
     with open(fname, errors='ignore') as f:
         lines = f.readlines()
         errors += check_ls(fname, lines[0])
         for l in lines:
-            if doclines or l.strip().endswith('=begin'):
+            if doclines or l.strip().endswith(tok1):
                 doclines.append(l.rstrip())
-            if l.startswith('=end'):
+            if l.startswith(tok2):
                 break
         else:
             if doclines:
@@ -39,7 +41,8 @@ def check_file(fname):
             else:
                 print('Error: no documentation in: ' + fname)
             return 1
-    title, underline = [d for d in doclines if d and '=begin' not in d][:2]
+    title, underline = [d for d in doclines
+                        if d and '=begin' not in d and '[====[' not in d][:2]
     if underline != '=' * len(title):
         print('Error: title/underline mismatch:', fname, title, underline)
         errors += 1
@@ -55,9 +58,7 @@ def main():
     err = 0
     for root, _, files in os.walk('scripts'):
         for f in files:
-            # TODO: remove 3rdparty exemptions from checks
+            if f[-3:] in {'.rb', 'lua'}:
-            # Requires reading their CMakeLists to only apply to used scripts
-            if f[-3:] in {'.rb', 'lua'} and '3rdparty' not in root:
                 err += check_file(join(root, f))
     return err