Much improved script docs generator

Creates a single file for each kind of script (base, gui, fix...).  This
includes and correctly sorts content from any .lua or .rb file under the
scripts directory.  It's more robust and more readable than the old
style, and doesn't write anything in the submodules.  User-added scripts
will be seamlessly added, if they have a docs section and Sphinx is run.

docs/3rdparty.rst

@@ -1,12 +0,0 @@
-######################
-Complete Scripts Index
-######################
-
-This is the complete index of documentation pulled from script source files.
-
-.. toctree::
-   :glob:
-   :maxdepth: 2
-
-   /scripts/**include-all
-

docs/Scripts.rst

@@ -15,35 +15,27 @@ or from the init file.
 only be interrupted every 256 instructions. Use ``kill-lua force`` to interrupt
 the next instruction.
 
-Here's an index to documentation pulled from script source files:
+Here's the contents page for this document:
-
-.. toctree::
-   :glob:
-   :maxdepth: 2
 
-   /docs/3rdparty/
+.. contents::
 
 
-Here's the contents page for this document:
+The following pages document all the standard DFHack scripts.
 
-.. contents::
+.. toctree::
+   :glob:
+   :maxdepth: 2
 
-=================
+   /docs/_auto/*
-3rd-party scripts
-=================
 
-This listing is autogenerated from ``.rst`` files in the 3rdparty scripts directory.
 
 .. warning::
 
-    Because this documentation is pulled in from external sources,
+    The following documentation is pulled in from external ``README`` files.
-    it may not match the DFHack distribution exactly.
-
-    * Some scripts should have a prefix (eg listed as ``myscript``,
-      should be ``gui/myscript``) but don't.
-    * Some sections of documentation may refer to scripts which are
-      not distributed with DFHack.
     
+    - it may not match the DFHack docs style
+    - some scripts have incorrect names, or are documented but unavailable
+    - some scripts have entries here and are also included above
 
 .. toctree::
    :glob:

docs/conf.py

@@ -18,41 +18,71 @@ from io import open
 import os
 import shlex
 import sys
+import shutil
 
-from os import listdir
+
-from os.path import isfile, join, isdir, relpath
+def doc_dir(dirname, files):
-
+    """Yield (command, includepath) for each script in the directory."""
-currentDir = '@CMAKE_CURRENT_SOURCE_DIR@'
+    sdir = os.path.relpath(dirname, '..').replace('\\', '/').replace('../', '')
-if currentDir.startswith('@'):
-    # Not running in CMake
-    currentDir = '.'
-
-def makeIncludeAll(directory, extension):
-  outputFile = join(directory,'include-all.rst')
-  files = [ f for f in listdir(directory) if isfile(join(directory,f)) and f.endswith(extension) ]
-  files.sort()
-  dname = relpath(directory, relpath('..', currentDir)).replace('\\', '/')
-  while dname.startswith('..'):
-    dname = dname[3:]
-  TEMPLATE = '.. include:: /{}/{}\n   :start-after: BEGIN_DOCS\n   :end-before: END_DOCS'
-  out = []
     for f in files:
-    with open(join(directory,f), 'r', encoding='utf8') as fstream:
+        if f[-3:] not in {'lua', '.rb'}:
-        data = fstream.read().replace('\n','')
+            continue
-    if 'BEGIN_DOCS' in data:
+        with open(os.path.join(dirname, f), 'r', encoding='utf8') as fstream:
-      out.append(TEMPLATE.format(dname, f))
+            text = [l.rstrip() for l in fstream.readlines() if l.strip()]
-  if out:
+        command = None
-    # Only write the file if the index is not empty
+        for line in text:
-    with open(outputFile, 'w' if sys.version_info.major > 2 else 'wb') as outfile:
+            if line == len(line) * '=':
-      outfile.write(len(dname)*'=' + '\n' + dname + '\n' + len(dname)*'=' + '\n\n')
+                if command is not None:
-      outfile.write('\n\n'.join(out))
+                    yield command, sdir + '/' + f
-
+                break
-def makeAllIncludeAll(directory, extension):
+            command = line
-  for root, subdirs, files in os.walk(directory):
+        # later, print an error for missing docs here
-    if isdir(root):
+
-      makeIncludeAll(root, extension)
+
-
+def document_scripts():
-makeAllIncludeAll(currentDir + '/scripts', '.lua')
+    """Autodoc for files with the magic script documentation marker strings.
+
+    Creates a file for eack kind of script (base/devel/fix/gui/modtools)
+    with all the ".. include::" directives to pull out docs between the
+    magic strings.
+    """
+    # First, we collect the commands and paths to include in our docs
+    scripts = []
+    for root, _, files in os.walk('../scripts'):
+        scripts.extend(doc_dir(root, files))
+    # Next we split by type and create include directives sorted by command
+    kinds = {'base': [], 'devel': [], 'fix': [], 'gui': [], 'modtools': []}
+    for s in scripts:
+        k_fname = s[0].split('/', 1)
+        if len(k_fname) == 1:
+            kinds['base'].append(s)
+        else:
+            kinds.get(k_fname[0], []).append(s)
+    template = ('.. include:: /{}\n   :start-after: BEGIN_DOCS\n'
+                '   :end-before: END_DOCS')
+    for key, value in kinds.items():
+        kinds[key] = [
+            template.format(x[1]) for x in sorted(value, key=lambda x: x[0])]
+    # Finally, we write our _auto/* files for each kind of script
+    if not os.path.isdir('_auto'):
+        os.mkdir('_auto')
+    head = {
+        'base': 'Basic Scripts',
+        'devel': 'Development Scripts',
+        'fix': 'Bugfixing Scripts',
+        'gui': 'GUI Scripts',
+        'modtools': 'Scripts for Modders'}
+    for k in head:
+        title = '{l}\n{t}\n{l}\n\n'.format(t=head[k], l=len(head[k])*'#')
+        mode = 'w' if sys.version_info.major > 2 else 'wb'
+        with open('_auto/{}.rst'.format(k), mode) as outfile:
+            outfile.write(title)
+            outfile.write('\n\n'.join(kinds[k]))
+
+
+# Actually call the docs generator
+document_scripts()
+
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the