From 4cdfcbaa9bccc4011261282ab320e550de3981a9 Mon Sep 17 00:00:00 2001 From: PeridexisErrant Date: Thu, 5 Nov 2015 00:23:45 +1100 Subject: [PATCH] Linting: improve script docs check, work off Travis Expands coverage of scripts linting, and improves checks for a correct title. Various fixes to make 'python travis/all.py' work for offline users; when the TRAVIS envvar is not set it now fails without an error. Minor cleanup in conf.py --- conf.py | 49 +++++++------------------------------- travis/lint.py | 5 +++- travis/script-in-readme.py | 43 +++++++++++++++++---------------- travis/script-syntax.py | 23 +++++++++++------- 4 files changed, 50 insertions(+), 70 deletions(-) diff --git a/conf.py b/conf.py index 81cfb78ef..5eb2c8177 100644 --- a/conf.py +++ b/conf.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # -# DFHack documentation build configuration file, created by -# sphinx-quickstart on Tue Sep 22 17:34:54 2015. +# DFHack documentation build configuration file # # This file is execfile()d with the current directory set to its # containing dir. @@ -18,25 +17,24 @@ from io import open import os import shlex import sys -import shutil +# -- Autodoc for DFhack scripts ------------------------------------------- + def doc_dir(dirname, files): """Yield (command, includepath) for each script in the directory.""" sdir = os.path.relpath(dirname, '.').replace('\\', '/').replace('../', '') for f in files: - if f[-3:] not in {'lua', '.rb'}: + if f[-3:] not in ('lua', '.rb'): continue with open(os.path.join(dirname, f), 'r', encoding='utf8') as fstream: text = [l.rstrip() for l in fstream.readlines() if l.strip()] command = None for line in text: - if line == len(line) * '=': - if command is not None: - yield command, sdir + '/' + f + if command and line == len(line) * '=': + yield command, sdir + '/' + f break command = line - # later, print an error for missing docs here def document_scripts(): @@ -89,11 +87,6 @@ def document_scripts(): document_scripts() -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) - # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. @@ -115,15 +108,12 @@ extlinks = { 'bug': ('http://www.bay12games.com/dwarves/mantisbt/view.php?id=%s', 'Bug ') } -# some aliases for link directives -extlinks['forum'] = extlinks['forums'] # Add any paths that contain templates here, relative to this directory. templates_path = [] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: -# source_suffix = ['.rst', '.md'] source_suffix = '.rst' # The encoding of source files. @@ -145,7 +135,7 @@ def get_version(): """Return the DFHack version string, from CMakeLists.txt""" version = release = '' try: - with open('../CMakeLists.txt') as f: + with open('CMakeLists.txt') as f: for s in f.readlines(): if fnmatch.fnmatch(s.upper(), 'SET(DF_VERSION "?.??.??")\n'): version = s.upper().replace('SET(DF_VERSION "', '') @@ -179,19 +169,9 @@ exclude_patterns = ['docs/_build/*', 'depends/*', 'scripts/3rdparty/*'] # documents. default_role = 'ref' -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' -# If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False - # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False @@ -245,7 +225,7 @@ html_static_path = ['docs/styles'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +html_last_updated_fmt = '%Y-%m-%d' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. @@ -288,19 +268,6 @@ html_use_index = False # base URL from which the finished HTML is served. #html_use_opensearch = '' -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Language to be used for generating the HTML full-text search index. -# Sphinx supports the following languages: -# 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' -# 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr' -#html_search_language = 'en' - -# The name of a javascript file (relative to the configuration directory) that -# implements a search results scorer. If empty, the default will be used. -#html_search_scorer = 'scorer.js' - # Output file base name for HTML help builder. htmlhelp_basename = 'DFHackdoc' diff --git a/travis/lint.py b/travis/lint.py index ce28f4be3..9b39f0149 100644 --- a/travis/lint.py +++ b/travis/lint.py @@ -65,8 +65,11 @@ class Linter(object): class NewlineLinter(Linter): msg = 'Contains DOS-style newlines' + def __init__(self): + # git supports newline conversion. Catch in CI, ignore on Windows. + self.ignore = sys.platform == 'win32' and not os.environ.get('TRAVIS') def check_line(self, line): - return '\r' not in line + return self.ignore or '\r' not in line def fix_line(self, line): return line.replace('\r', '') diff --git a/travis/script-in-readme.py b/travis/script-in-readme.py index b7d0577f3..a6804fb00 100644 --- a/travis/script-in-readme.py +++ b/travis/script-in-readme.py @@ -1,17 +1,19 @@ from io import open import os +from os.path import basename, dirname, join, splitext import sys -scriptdirs = ( - 'scripts', - #'scripts/devel', # devel scripts don't have to be documented - 'scripts/fix', - 'scripts/gui', - 'scripts/modtools') + +def expected_cmd(path): + """Get the command from the name of a script.""" + dname, fname = basename(dirname(path)), splitext(basename(path))[0] + if dname in ('devel', 'fix', 'gui', 'modtools'): + return dname + '/' + fname + return fname def check_file(fname): - doclines = [] + errors, doclines = 0, [] with open(fname, errors='ignore') as f: for l in f.readlines(): if doclines or l.strip().endswith('=begin'): @@ -27,23 +29,24 @@ def check_file(fname): title, underline = doclines[2], doclines[3] if underline != '=' * len(title): print('Error: title/underline mismatch:', fname, title, underline) - return 1 - start = fname.split('/')[-2] - if start != 'scripts' and not title.startswith(start): - print('Error: title is missing start string: {} {} {}'.format(fname, start, title)) - return 1 - return 0 + errors += 1 + if title != expected_cmd(fname): + print('Warning: expected script title {}, got {}'.format( + expected_cmd(fname), title)) + errors += 1 + return errors def main(): """Check that all DFHack scripts include documentation (not 3rdparty)""" - errors = 0 - for path in scriptdirs: - for f in os.listdir(path): - f = path + '/' + f - if os.path.isfile(f) and f[-3:] in {'.rb', 'lua'}: - errors += check_file(f) - return errors + err = 0 + for root, _, files in os.walk('scripts'): + for f in files: + # TODO: remove 3rdparty exemptions from checks + # Requires reading their CMakeLists to only apply to used scripts + if f[-3:] in {'.rb', 'lua'} and '3rdparty' not in root: + err += check_file(join(root, f)) + return err if __name__ == '__main__': diff --git a/travis/script-syntax.py b/travis/script-syntax.py index 984ff1755..0b977f59d 100644 --- a/travis/script-syntax.py +++ b/travis/script-syntax.py @@ -1,15 +1,12 @@ -import argparse, os, sys, subprocess +import argparse +import os +import subprocess +import sys -parser = argparse.ArgumentParser() -parser.add_argument('--path', default='.', help='Root directory') -parser.add_argument('--ext', help='Script extension', required=True) -parser.add_argument('--cmd', help='Command', required=True) -args = parser.parse_args() def main(): root_path = os.path.abspath(args.path) cmd = args.cmd.split(' ') - ext = '.' + args.ext if not os.path.exists(root_path): print('Nonexistent path: %s' % root_path) sys.exit(2) @@ -19,14 +16,24 @@ def main(): if '.git' in parts or 'depends' in parts: continue for filename in filenames: - if not filename.endswith(ext): + if not filename.endswith(args.ext): continue full_path = os.path.join(cur, filename) try: subprocess.check_output(cmd + [full_path]) except subprocess.CalledProcessError: err = True + except IOError: + # catch error if not in Travis and Lua/Ruby is not available + if os.environ.get('TRAVIS', False): + raise sys.exit(int(err)) + if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--path', default='.', help='Root directory') + parser.add_argument('--ext', help='Script extension', required=True) + parser.add_argument('--cmd', help='Command', required=True) + args = parser.parse_args() main()