Linting: improve script docs check, work off Travis

Expands coverage of scripts linting, and improves checks for a correct
title.  Various fixes to make 'python travis/all.py' work for offline
users; when the TRAVIS envvar is not set it now fails without an error.

Minor cleanup in conf.py

conf.py

@@ -1,8 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
-# DFHack documentation build configuration file, created by
+# DFHack documentation build configuration file
-# sphinx-quickstart on Tue Sep 22 17:34:54 2015.
 #
 # This file is execfile()d with the current directory set to its
 # containing dir.
@@ -18,25 +17,24 @@ from io import open
 import os
 import shlex
 import sys
-import shutil
 
 
+# -- Autodoc for DFhack scripts -------------------------------------------
+
 def doc_dir(dirname, files):
     """Yield (command, includepath) for each script in the directory."""
     sdir = os.path.relpath(dirname, '.').replace('\\', '/').replace('../', '')
     for f in files:
-        if f[-3:] not in {'lua', '.rb'}:
+        if f[-3:] not in ('lua', '.rb'):
             continue
         with open(os.path.join(dirname, f), 'r', encoding='utf8') as fstream:
             text = [l.rstrip() for l in fstream.readlines() if l.strip()]
         command = None
         for line in text:
-            if line == len(line) * '=':
+            if command and line == len(line) * '=':
-                if command is not None:
+                yield command, sdir + '/' + f
-                    yield command, sdir + '/' + f
                 break
             command = line
-        # later, print an error for missing docs here
 
 
 def document_scripts():
@@ -89,11 +87,6 @@ def document_scripts():
 document_scripts()
 
 
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-#sys.path.insert(0, os.path.abspath('.'))
-
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -115,15 +108,12 @@ extlinks = {
     'bug': ('http://www.bay12games.com/dwarves/mantisbt/view.php?id=%s',
             'Bug ')
 }
-# some aliases for link directives
-extlinks['forum'] = extlinks['forums']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = []
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
-# source_suffix = ['.rst', '.md']
 source_suffix = '.rst'
 
 # The encoding of source files.
@@ -145,7 +135,7 @@ def get_version():
     """Return the DFHack version string, from CMakeLists.txt"""
     version = release = ''
     try:
-        with open('../CMakeLists.txt') as f:
+        with open('CMakeLists.txt') as f:
             for s in f.readlines():
                 if fnmatch.fnmatch(s.upper(), 'SET(DF_VERSION "?.??.??")\n'):
                     version = s.upper().replace('SET(DF_VERSION "', '')
@@ -179,19 +169,9 @@ exclude_patterns = ['docs/_build/*', 'depends/*', 'scripts/3rdparty/*']
 # documents.
 default_role = 'ref'
 
-# If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
-
-# If true, the current module name will be prepended to all description
-# unit titles (such as .. function::).
-#add_module_names = True
-
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = 'sphinx'
 
-# If true, keep warnings as "system message" paragraphs in the built documents.
-#keep_warnings = False
-
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = False
 
@@ -245,7 +225,7 @@ html_static_path = ['docs/styles']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
-#html_last_updated_fmt = '%b %d, %Y'
+html_last_updated_fmt = '%Y-%m-%d'
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.
@@ -288,19 +268,6 @@ html_use_index = False
 # base URL from which the finished HTML is served.
 #html_use_opensearch = ''
 
-# This is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = None
-
-# Language to be used for generating the HTML full-text search index.
-# Sphinx supports the following languages:
-#   'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja'
-#   'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr'
-#html_search_language = 'en'
-
-# The name of a javascript file (relative to the configuration directory) that
-# implements a search results scorer. If empty, the default will be used.
-#html_search_scorer = 'scorer.js'
-
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'DFHackdoc'
 

travis/lint.py

@@ -65,8 +65,11 @@ class Linter(object):
 
 class NewlineLinter(Linter):
     msg = 'Contains DOS-style newlines'
+    def __init__(self):
+        # git supports newline conversion.  Catch in CI, ignore on Windows.
+        self.ignore = sys.platform == 'win32' and not os.environ.get('TRAVIS')
     def check_line(self, line):
-        return '\r' not in line
+        return self.ignore or '\r' not in line
     def fix_line(self, line):
         return line.replace('\r', '')
 

travis/script-in-readme.py

@@ -1,17 +1,19 @@
 from io import open
 import os
+from os.path import basename, dirname, join, splitext
 import sys
 
-scriptdirs = (
+
-    'scripts',
+def expected_cmd(path):
-    #'scripts/devel',  # devel scripts don't have to be documented
+    """Get the command from the name of a script."""
-    'scripts/fix',
+    dname, fname = basename(dirname(path)), splitext(basename(path))[0]
-    'scripts/gui',
+    if dname in ('devel', 'fix', 'gui', 'modtools'):
-    'scripts/modtools')
+        return dname + '/' + fname
+    return fname
 
 
 def check_file(fname):
-    doclines = []
+    errors, doclines = 0, []
     with open(fname, errors='ignore') as f:
         for l in f.readlines():
             if doclines or l.strip().endswith('=begin'):
@@ -27,23 +29,24 @@ def check_file(fname):
     title, underline = doclines[2], doclines[3]
     if underline != '=' * len(title):
         print('Error: title/underline mismatch:', fname, title, underline)
-        return 1
+        errors += 1
-    start = fname.split('/')[-2]
+    if title != expected_cmd(fname):
-    if start != 'scripts' and not title.startswith(start):
+        print('Warning: expected script title {}, got {}'.format(
-        print('Error: title is missing start string: {} {} {}'.format(fname, start, title))
+              expected_cmd(fname), title))
-        return 1
+        errors += 1
-    return 0
+    return errors
 
 
 def main():
     """Check that all DFHack scripts include documentation (not 3rdparty)"""
-    errors = 0
+    err = 0
-    for path in scriptdirs:
+    for root, _, files in os.walk('scripts'):
-        for f in os.listdir(path):
+        for f in files:
-            f = path + '/' + f
+            # TODO: remove 3rdparty exemptions from checks
-            if os.path.isfile(f) and f[-3:] in {'.rb', 'lua'}:
+            # Requires reading their CMakeLists to only apply to used scripts
-                errors += check_file(f)
+            if f[-3:] in {'.rb', 'lua'} and '3rdparty' not in root:
-    return errors
+                err += check_file(join(root, f))
+    return err
 
 
 if __name__ == '__main__':

travis/script-syntax.py

@@ -1,15 +1,12 @@
-import argparse, os, sys, subprocess
+import argparse
+import os
+import subprocess
+import sys
 
-parser = argparse.ArgumentParser()
-parser.add_argument('--path', default='.', help='Root directory')
-parser.add_argument('--ext', help='Script extension', required=True)
-parser.add_argument('--cmd', help='Command', required=True)
-args = parser.parse_args()
 
 def main():
     root_path = os.path.abspath(args.path)
     cmd = args.cmd.split(' ')
-    ext = '.' + args.ext
     if not os.path.exists(root_path):
         print('Nonexistent path: %s' % root_path)
         sys.exit(2)
@@ -19,14 +16,24 @@ def main():
         if '.git' in parts or 'depends' in parts:
             continue
         for filename in filenames:
-            if not filename.endswith(ext):
+            if not filename.endswith(args.ext):
                 continue
             full_path = os.path.join(cur, filename)
             try:
                 subprocess.check_output(cmd + [full_path])
             except subprocess.CalledProcessError:
                 err = True
+            except IOError:
+                # catch error if not in Travis and Lua/Ruby is not available
+                if os.environ.get('TRAVIS', False):
+                    raise
     sys.exit(int(err))
 
+
 if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--path', default='.', help='Root directory')
+    parser.add_argument('--ext', help='Script extension', required=True)
+    parser.add_argument('--cmd', help='Command', required=True)
+    args = parser.parse_args()
     main()