diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 869ca407e..14361daeb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -50,7 +50,7 @@ jobs: - name: Set up environment id: env_setup run: | - DF_VERSION="$(sh travis/get-df-version.sh)" + DF_VERSION="$(sh ci/get-df-version.sh)" echo "::set-output name=df_version::${DF_VERSION}" echo "DF_VERSION=${DF_VERSION}" >> $GITHUB_ENV echo "DF_FOLDER=${HOME}/DF/${DF_VERSION}/df_linux" >> $GITHUB_ENV @@ -61,7 +61,7 @@ jobs: key: ${{ steps.env_setup.outputs.df_version }} - name: Download DF run: | - sh travis/download-df.sh + sh ci/download-df.sh - name: Build DFHack env: CC: gcc-${{ matrix.gcc }} @@ -84,8 +84,8 @@ jobs: run: | export TERM=dumb mv "$DF_FOLDER"/dfhack.init-example "$DF_FOLDER"/dfhack.init - script -qe -c "python travis/run-tests.py --headless --keep-status \"$DF_FOLDER\"" - python travis/check-rpc.py "$DF_FOLDER/dfhack-rpc.txt" + script -qe -c "python ci/run-tests.py --headless --keep-status \"$DF_FOLDER\"" + python ci/check-rpc.py "$DF_FOLDER/dfhack-rpc.txt" mkdir -p artifacts cp "$DF_FOLDER/test_status.json" "$DF_FOLDER"/*.log artifacts - name: Upload test artifacts @@ -147,23 +147,23 @@ jobs: # don't need tags here - name: Check whitespace run: | - python travis/lint.py + python ci/lint.py --git-only --github-actions - name: Check Authors.rst if: success() || failure() run: | - python travis/authors-rst.py + python ci/authors-rst.py - name: Check for missing documentation if: success() || failure() run: | - python travis/script-docs.py + python ci/script-docs.py - name: Check Lua syntax if: success() || failure() run: | - python travis/script-syntax.py --ext=lua --cmd="luac5.3 -p" --github-actions + python ci/script-syntax.py --ext=lua --cmd="luac5.3 -p" --github-actions - name: Check Ruby syntax if: success() || failure() run: | - python travis/script-syntax.py --ext=rb --cmd="ruby -c" --github-actions + python ci/script-syntax.py --ext=rb --cmd="ruby -c" --github-actions check-pr: runs-on: ubuntu-latest diff --git a/ci/authors-rst.py b/ci/authors-rst.py new file mode 100755 index 000000000..076f7f226 --- /dev/null +++ b/ci/authors-rst.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +""" Overly-complicated script to check formatting/sorting in Authors.rst """ + +import os, re, sys + +def main(): + success = [True] + def error(line, msg, **kwargs): + info = '' + for k in kwargs: + info += ' %s %s:' % (k, kwargs[k]) + print('line %i:%s %s' % (line, info, msg)) + if os.environ.get('GITHUB_ACTIONS'): + print('::error file=docs/Authors.rst,line=%i::%s %s' % (line, info.lstrip(), msg)) + success[0] = False + with open('docs/Authors.rst', 'rb') as f: + lines = list(map(lambda line: line.decode('utf8').replace('\n', ''), f.readlines())) + + if lines[1].startswith('='): + if len(lines[0]) != len(lines[1]): + error(2, 'Length of header does not match underline') + if lines[1].replace('=', ''): + error(2, 'Invalid header') + + first_div_index = list(filter(lambda pair: pair[1].startswith('==='), enumerate(lines[2:])))[0][0] + 2 + first_div = lines[first_div_index] + div_indices = [] + for i, line in enumerate(lines[first_div_index:]): + line_number = i + first_div_index + 1 + if '\t' in line: + error(line_number, 'contains tabs') + if line.startswith('==='): + div_indices.append(i + first_div_index) + if not re.match(r'^=+( =+)+$', line): + error(line_number, 'bad table divider') + if line != lines[first_div_index]: + error(line_number, 'malformed table divider') + if len(div_indices) < 3: + error(len(lines), 'missing table divider(s)') + for i in div_indices[3:]: + error(i + 1, 'extra table divider') + + col_ranges = [] + i = 0 + while True: + j = first_div.find(' ', i) + col_ranges.append(slice(i, j if j > 0 else None)) + if j == -1: + break + i = j + 1 + + for i, line in enumerate(lines[div_indices[1] + 1:div_indices[2]]): + line_number = i + div_indices[1] + 2 + for c, col in enumerate(col_ranges): + cell = line[col] + if cell.startswith(' '): + error(line_number, 'text does not start in correct location', column=c+1) + # check for text extending into next column if this isn't the last column + if col.stop is not None and col.stop < len(line) and line[col.stop] != ' ': + error(line_number, 'text extends into next column', column=c+1) + if i > 0: + prev_line = lines[div_indices[1] + i] + if line.lower()[col_ranges[0]] < prev_line.lower()[col_ranges[0]]: + error(line_number, 'not sorted: should come before line %i ("%s" before "%s")' % + (line_number - 1, line[col_ranges[0]].rstrip(' '), prev_line[col_ranges[0]].rstrip(' '))) + + return success[0] + +if __name__ == '__main__': + sys.exit(int(not main())) diff --git a/travis/build-lua.sh b/ci/build-lua.sh old mode 100644 new mode 100755 similarity index 100% rename from travis/build-lua.sh rename to ci/build-lua.sh diff --git a/ci/check-rpc.py b/ci/check-rpc.py new file mode 100755 index 000000000..aba3e3811 --- /dev/null +++ b/ci/check-rpc.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +import glob +import sys + +actual = {'': {}} + +with open(sys.argv[1]) as f: + plugin_name = '' + for line in f: + line = line.rstrip() + if line.startswith('// Plugin: '): + plugin_name = line.split(' ')[2] + if plugin_name not in actual: + actual[plugin_name] = {} + elif line.startswith('// RPC '): + parts = line.split(' ') + actual[plugin_name][parts[2]] = (parts[4], parts[6]) + +expected = {'': {}} + +for p in glob.iglob('library/proto/*.proto'): + with open(p) as f: + for line in f: + line = line.rstrip() + if line.startswith('// RPC '): + parts = line.split(' ') + expected[''][parts[2]] = (parts[4], parts[6]) + +for p in glob.iglob('plugins/proto/*.proto'): + plugin_name = '' + with open(p) as f: + for line in f: + line = line.rstrip() + if line.startswith('// Plugin: '): + plugin_name = line.split(' ')[2] + if plugin_name not in expected: + expected[plugin_name] = {} + break + + if plugin_name == '': + continue + + with open(p) as f: + for line in f: + line = line.rstrip() + if line.startswith('// RPC '): + parts = line.split(' ') + expected[plugin_name][parts[2]] = (parts[4], parts[6]) + +error_count = 0 + +for plugin_name in actual: + methods = actual[plugin_name] + + if plugin_name not in expected: + print('Missing documentation for plugin proto files: ' + plugin_name) + print('Add the following lines:') + print('// Plugin: ' + plugin_name) + error_count += 1 + for m in methods: + io = methods[m] + print('// RPC ' + m + ' : ' + io[0] + ' -> ' + io[1]) + error_count += 1 + else: + missing = [] + wrong = [] + for m in methods: + io = methods[m] + if m in expected[plugin_name]: + if expected[plugin_name][m] != io: + wrong.append('// RPC ' + m + ' : ' + io[0] + ' -> ' + io[1]) + else: + missing.append('// RPC ' + m + ' : ' + io[0] + ' -> ' + io[1]) + + if len(missing) > 0: + print('Incomplete documentation for ' + ('core' if plugin_name == '' else 'plugin "' + plugin_name + '"') + ' proto files. Add the following lines:') + for m in missing: + print(m) + error_count += 1 + + if len(wrong) > 0: + print('Incorrect documentation for ' + ('core' if plugin_name == '' else 'plugin "' + plugin_name + '"') + ' proto files. Replace the following comments:') + for m in wrong: + print(m) + error_count += 1 + +for plugin_name in expected: + methods = expected[plugin_name] + + if plugin_name not in actual: + print('Incorrect documentation for plugin proto files: ' + plugin_name) + print('The following methods are documented, but the plugin does not provide any RPC methods:') + for m in methods: + io = methods[m] + print('// RPC ' + m + ' : ' + io[0] + ' -> ' + io[1]) + error_count += 1 + else: + missing = [] + for m in methods: + io = methods[m] + if m not in actual[plugin_name]: + missing.append('// RPC ' + m + ' : ' + io[0] + ' -> ' + io[1]) + + if len(missing) > 0: + print('Incorrect documentation for ' + ('core' if plugin_name == '' else 'plugin "' + plugin_name + '"') + ' proto files. Remove the following lines:') + for m in missing: + print(m) + error_count += 1 + +sys.exit(min(100, error_count)) diff --git a/ci/download-df.sh b/ci/download-df.sh new file mode 100755 index 000000000..49dcedea7 --- /dev/null +++ b/ci/download-df.sh @@ -0,0 +1,56 @@ +#!/bin/sh + +set -e + +tardest="df.tar.bz2" + +selfmd5=$(openssl md5 < "$0") +echo $selfmd5 + +cd "$(dirname "$0")" +echo "DF_VERSION: $DF_VERSION" +echo "DF_FOLDER: $DF_FOLDER" +mkdir -p "$DF_FOLDER" +# back out of df_linux +cd "$DF_FOLDER/.." + +if [ -f receipt ]; then + if [ "$selfmd5" != "$(cat receipt)" ]; then + echo "download-df.sh changed; removing DF" + rm receipt + else + echo "Already downloaded $DF_VERSION" + fi +fi + +if [ ! -f receipt ]; then + rm -f "$tardest" + minor=$(echo "$DF_VERSION" | cut -d. -f2) + patch=$(echo "$DF_VERSION" | cut -d. -f3) + url="http://www.bay12games.com/dwarves/df_${minor}_${patch}_linux.tar.bz2" + echo Downloading + while read url; do + echo "Attempting download: ${url}" + if wget -v "$url" -O "$tardest"; then + break + fi + done < receipt +ls diff --git a/ci/get-df-version.sh b/ci/get-df-version.sh new file mode 100755 index 000000000..24386252d --- /dev/null +++ b/ci/get-df-version.sh @@ -0,0 +1,4 @@ +#!/bin/sh +cd "$(dirname "$0")" +cd .. +grep -i 'set(DF_VERSION' CMakeLists.txt | perl -ne 'print "$&\n" if /[\d\.]+/' diff --git a/ci/lint-check.txt b/ci/lint-check.txt new file mode 100644 index 000000000..bdfa74365 --- /dev/null +++ b/ci/lint-check.txt @@ -0,0 +1,32 @@ +# Files that lint.py should check + +*.bash +*.bat +*.c +*.cc +*.cmake +*.cpp +*.css +*.gitignore +*.h +*.hh +*.hpp +*.in +*.inc +*.init +*.init-example +*.js +*.lua +*.manifest +*.md +*.mm +*.pl +*.proto +*.py +*.rb +*.rst +*.sh +*.txt +*.vbs +*.yaml +*.yml diff --git a/ci/lint-ignore.txt b/ci/lint-ignore.txt new file mode 100644 index 000000000..d384b80f3 --- /dev/null +++ b/ci/lint-ignore.txt @@ -0,0 +1,22 @@ +# Files that lint.py should ignore + +.git/* + +# Old files exempt from checks for now +plugins/isoworld/*.txt +plugins/raw/*.txt +plugins/stonesense/*.txt + +# Generated files +*.pb.h +build*/* +docs/_* +docs/html/* +docs/pdf/* +library/include/df/* + +# Dependencies that we don't control +depends/* +plugins/isoworld/agui/* +plugins/isoworld/allegro/* +plugins/stonesense/allegro/* diff --git a/ci/lint.py b/ci/lint.py new file mode 100755 index 000000000..b2fb8e647 --- /dev/null +++ b/ci/lint.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +import argparse +import fnmatch +import re +import os +import subprocess +import sys + +DFHACK_ROOT = os.path.normpath(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +def load_pattern_files(paths): + patterns = [] + for p in paths: + with open(p) as f: + for line in f.readlines(): + line = line.strip() + if line and not line.startswith('#'): + patterns.append(line) + return patterns + +def valid_file(rel_path, check_patterns, ignore_patterns): + return ( + any(fnmatch.fnmatch(rel_path, pattern) for pattern in check_patterns) + and not any(fnmatch.fnmatch(rel_path, pattern) for pattern in ignore_patterns) + ) + +success = True +def error(msg=None): + global success + success = False + if msg: + sys.stderr.write(msg + '\n') + +def format_lines(lines, total): + if len(lines) == total - 1: + return 'entire file' + if not len(lines): + # should never happen + return 'nowhere' + if len(lines) == 1: + return 'line %i' % lines[0] + s = 'lines ' + range_start = range_end = lines[0] + for i, line in enumerate(lines): + if line > range_end + 1: + if range_start == range_end: + s += ('%i, ' % range_end) + else: + s += ('%i-%i, ' % (range_start, range_end)) + range_start = range_end = line + if i == len(lines) - 1: + s += ('%i' % line) + else: + range_end = line + if i == len(lines) - 1: + s += ('%i-%i, ' % (range_start, range_end)) + return s.rstrip(' ').rstrip(',') + +class LinterError(Exception): + def __init__(self, message, lines, total_lines): + self.message = message + self.lines = lines + self.total_lines = total_lines + + def __str__(self): + return '%s: %s' % (self.message, format_lines(self.lines, self.total_lines)) + + def github_actions_workflow_command(self, filename): + first_line = self.lines[0] if self.lines else 1 + return '::error file=%s,line=%i::%s' % (filename, first_line, self) + +class Linter(object): + ignore = False + def check(self, lines): + failures = [] + for i, line in enumerate(lines): + if not self.check_line(line): + failures.append(i + 1) + if len(failures): + raise LinterError(self.msg, failures, len(lines)) + + def fix(self, lines): + for i in range(len(lines)): + lines[i] = self.fix_line(lines[i]) + + +class NewlineLinter(Linter): + msg = 'Contains DOS-style newlines' + # git supports newline conversion. Catch in CI, ignore on Windows. + ignore = os.linesep != '\n' and not os.environ.get('CI') + def check_line(self, line): + return '\r' not in line + def fix_line(self, line): + return line.replace('\r', '') + +class TrailingWhitespaceLinter(Linter): + msg = 'Contains trailing whitespace' + def check_line(self, line): + line = line.replace('\r', '').replace('\n', '') + return not line.strip() or line == line.rstrip('\t ') + def fix_line(self, line): + return line.rstrip('\t ') + +class TabLinter(Linter): + msg = 'Contains tabs' + def check_line(self, line): + return '\t' not in line + def fix_line(self, line): + return line.replace('\t', ' ') + +linters = [cls() for cls in Linter.__subclasses__() if not cls.ignore] + +def walk_all(root_path): + for cur, dirnames, filenames in os.walk(root_path): + for filename in filenames: + full_path = os.path.join(cur, filename) + yield full_path + +def walk_git_files(root_path): + p = subprocess.Popen(['git', '-C', root_path, 'ls-files', root_path], stdout=subprocess.PIPE) + for line in p.stdout.readlines(): + path = line.decode('utf-8').strip() + full_path = os.path.join(root_path, path) + yield full_path + if p.wait() != 0: + raise RuntimeError('git exited with %r' % p.returncode) + +def main(args): + root_path = os.path.abspath(args.path) + if not os.path.exists(args.path): + print('Nonexistent path: %s' % root_path) + sys.exit(2) + + check_patterns = load_pattern_files(args.check_patterns) + ignore_patterns = load_pattern_files(args.ignore_patterns) + + walk_iter = walk_all + if args.git_only: + walk_iter = walk_git_files + + for full_path in walk_iter(root_path): + rel_path = full_path.replace(root_path, '').replace('\\', '/').lstrip('/') + if not valid_file(rel_path, check_patterns, ignore_patterns): + continue + if args.verbose: + print('Checking:', rel_path) + lines = [] + with open(full_path, 'rb') as f: + lines = f.read().split(b'\n') + for i, line in enumerate(lines): + try: + lines[i] = line.decode('utf-8') + except UnicodeDecodeError: + msg_params = (rel_path, i + 1, 'Invalid UTF-8 (other errors will be ignored)') + error('%s:%i: %s' % msg_params) + if args.github_actions: + print('::error file=%s,line=%i::%s' % msg_params) + lines[i] = '' + for linter in linters: + try: + linter.check(lines) + except LinterError as e: + error('%s: %s' % (rel_path, e)) + if args.github_actions: + print(e.github_actions_workflow_command(rel_path)) + if args.fix: + linter.fix(lines) + contents = '\n'.join(lines) + with open(full_path, 'wb') as f: + f.write(contents.encode('utf-8')) + + if success: + print('All linters completed successfully') + sys.exit(0) + else: + sys.exit(1) + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('path', nargs='?', default='.', + help='Path to scan (default: current directory)') + parser.add_argument('--fix', action='store_true', + help='Attempt to modify files in-place to fix identified issues') + parser.add_argument('--git-only', action='store_true', + help='Only check files tracked by git') + parser.add_argument('--github-actions', action='store_true', + help='Enable GitHub Actions workflow command output') + parser.add_argument('-v', '--verbose', action='store_true', + help='Log files as they are checked') + parser.add_argument('--check-patterns', action='append', + default=[os.path.join(DFHACK_ROOT, 'ci', 'lint-check.txt')], + help='File(s) containing filename patterns to check') + parser.add_argument('--ignore-patterns', action='append', + default=[os.path.join(DFHACK_ROOT, 'ci', 'lint-ignore.txt')], + help='File(s) containing filename patterns to ignore') + args = parser.parse_args() + main(args) diff --git a/ci/run-tests.py b/ci/run-tests.py new file mode 100755 index 000000000..132631903 --- /dev/null +++ b/ci/run-tests.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +import argparse +import enum +import json +import os +import re +import shutil +import subprocess +import sys + +parser = argparse.ArgumentParser() +parser.add_argument('df_folder', help='DF base folder') +parser.add_argument('--headless', action='store_true', + help='Run without opening DF window (requires non-Windows)') +parser.add_argument('--keep-status', action='store_true', + help='Do not delete final status file') +parser.add_argument('--no-quit', action='store_true', + help='Do not quit DF when done') +parser.add_argument('--test-dir', '--test-folder', + help='Base test folder (default: df_folder/test)') +parser.add_argument('-t', '--test', dest='tests', nargs='+', + help='Test(s) to run (Lua patterns accepted)') +args = parser.parse_args() + +if (not sys.stdin.isatty() or not sys.stdout.isatty() or not sys.stderr.isatty()) and not args.headless: + print('WARN: no TTY detected, enabling headless mode') + args.headless = True + +if args.test_dir is not None: + args.test_dir = os.path.normpath(os.path.join(os.getcwd(), args.test_dir)) + if not os.path.isdir(args.test_dir): + print('ERROR: invalid test folder: %r' % args.test_dir) + +MAX_TRIES = 5 + +dfhack = 'Dwarf Fortress.exe' if sys.platform == 'win32' else './dfhack' +test_status_file = 'test_status.json' + +class TestStatus(enum.Enum): + PENDING = 'pending' + PASSED = 'passed' + FAILED = 'failed' + +def get_test_status(): + if os.path.isfile(test_status_file): + with open(test_status_file) as f: + return {k: TestStatus(v) for k, v in json.load(f).items()} + +def change_setting(content, setting, value): + return '[' + setting + ':' + value + ']\n' + re.sub( + r'\[' + setting + r':.+?\]', '(overridden)', content, flags=re.IGNORECASE) + +os.chdir(args.df_folder) +if os.path.exists(test_status_file): + os.remove(test_status_file) + +print('Backing up init.txt to init.txt.orig') +init_txt_path = 'data/init/init.txt' +shutil.copyfile(init_txt_path, init_txt_path + '.orig') +with open(init_txt_path) as f: + init_contents = f.read() +init_contents = change_setting(init_contents, 'INTRO', 'NO') +init_contents = change_setting(init_contents, 'SOUND', 'NO') +init_contents = change_setting(init_contents, 'WINDOWED', 'YES') +init_contents = change_setting(init_contents, 'WINDOWEDX', '80') +init_contents = change_setting(init_contents, 'WINDOWEDY', '25') +init_contents = change_setting(init_contents, 'FPS', 'YES') +if args.headless: + init_contents = change_setting(init_contents, 'PRINT_MODE', 'TEXT') + +test_init_file = 'dfhackzzz_test.init' # Core sorts these alphabetically +with open(test_init_file, 'w') as f: + f.write(''' + devel/dump-rpc dfhack-rpc.txt + :lua dfhack.internal.addScriptPath(dfhack.getHackPath()) + test --resume --modes=none,title "lua scr.breakdown_level=df.interface_breakdown_types.%s" + ''' % ('NONE' if args.no_quit else 'QUIT')) + +test_config_file = 'test_config.json' +with open(test_config_file, 'w') as f: + json.dump({ + 'test_dir': args.test_dir, + 'tests': args.tests, + }, f) + +try: + with open(init_txt_path, 'w') as f: + f.write(init_contents) + + tries = 0 + while True: + status = get_test_status() + if status is not None: + if all(s != TestStatus.PENDING for s in status.values()): + print('Done!') + sys.exit(int(any(s != TestStatus.PASSED for s in status.values()))) + elif tries > 0: + print('ERROR: Could not read status file') + sys.exit(2) + + tries += 1 + print('Starting DF: #%i' % (tries)) + if tries > MAX_TRIES: + print('ERROR: Too many tries - aborting') + sys.exit(1) + + if args.headless: + os.environ['DFHACK_HEADLESS'] = '1' + os.environ['DFHACK_DISABLE_CONSOLE'] = '1' + + process = subprocess.Popen([dfhack], + stdin=subprocess.PIPE if args.headless else sys.stdin, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + _, err = process.communicate() + if err: + print('WARN: DF produced stderr: ' + repr(err[:5000])) + if process.returncode != 0: + print('ERROR: DF exited with ' + repr(process.returncode)) +finally: + print('\nRestoring original init.txt') + shutil.copyfile(init_txt_path + '.orig', init_txt_path) + if os.path.isfile(test_init_file): + os.remove(test_init_file) + if not args.keep_status and os.path.isfile(test_status_file): + os.remove(test_status_file) + print('Cleanup done') diff --git a/ci/script-docs.py b/ci/script-docs.py new file mode 100755 index 000000000..71d7f37b2 --- /dev/null +++ b/ci/script-docs.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +import os +from os.path import basename, dirname, join, splitext +import sys + +SCRIPT_PATH = sys.argv[1] if len(sys.argv) > 1 else 'scripts' +IS_GITHUB_ACTIONS = bool(os.environ.get('GITHUB_ACTIONS')) + +def expected_cmd(path): + """Get the command from the name of a script.""" + dname, fname = basename(dirname(path)), splitext(basename(path))[0] + if dname in ('devel', 'fix', 'gui', 'modtools'): + return dname + '/' + fname + return fname + + +def check_ls(fname, line): + """Check length & existence of leading comment for "ls" builtin command.""" + line = line.strip() + comment = '--' if fname.endswith('.lua') else '#' + if '[====[' in line or not line.startswith(comment): + print_error('missing leading comment (requred for `ls`)', fname) + return 1 + return 0 + + +def print_error(message, filename, line=None): + if not isinstance(line, int): + line = 1 + print('Error: %s:%i: %s' % (filename, line, message)) + if IS_GITHUB_ACTIONS: + print('::error file=%s,line=%i::%s' % (filename, line, message)) + + +def check_file(fname): + errors, doclines = 0, [] + tok1, tok2 = ('=begin', '=end') if fname.endswith('.rb') else \ + ('[====[', ']====]') + doc_start_line = None + with open(fname, errors='ignore') as f: + lines = f.readlines() + if not lines: + print_error('empty file', fname) + return 1 + errors += check_ls(fname, lines[0]) + for i, l in enumerate(lines): + if doclines or l.strip().endswith(tok1): + if not doclines: + doc_start_line = i + 1 + doclines.append(l.rstrip()) + if l.startswith(tok2): + break + else: + if doclines: + print_error('docs start but do not end', fname, doc_start_line) + else: + print_error('no documentation found', fname) + return 1 + + if not doclines: + print_error('missing or malformed documentation', fname) + return 1 + + title, underline = [d for d in doclines + if d and '=begin' not in d and '[====[' not in d][:2] + title_line = doc_start_line + doclines.index(title) + expected_underline = '=' * len(title) + if underline != expected_underline: + print_error('title/underline mismatch: expected {!r}, got {!r}'.format( + expected_underline, underline), + fname, title_line + 1) + errors += 1 + if title != expected_cmd(fname): + print_error('expected script title {!r}, got {!r}'.format( + expected_cmd(fname), title), + fname, title_line) + errors += 1 + return errors + + +def main(): + """Check that all DFHack scripts include documentation""" + err = 0 + exclude = set(['internal', 'test']) + for root, dirs, files in os.walk(SCRIPT_PATH, topdown=True): + dirs[:] = [d for d in dirs if d not in exclude] + for f in files: + if f[-3:] in {'.rb', 'lua'}: + err += check_file(join(root, f)) + return err + + +if __name__ == '__main__': + sys.exit(min(100, main())) diff --git a/ci/script-syntax.py b/ci/script-syntax.py new file mode 100755 index 000000000..593a25a24 --- /dev/null +++ b/ci/script-syntax.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +import argparse +import os +import subprocess +import sys + + +def print_stderr(stderr, args): + if not args.github_actions: + sys.stderr.write(stderr + '\n') + return + + for line in stderr.split('\n'): + print(line) + parts = list(map(str.strip, line.split(':'))) + # e.g. luac prints "luac:" in front of messages, so find the first part + # containing the actual filename + for i in range(len(parts) - 1): + if parts[i].endswith('.' + args.ext) and parts[i + 1].isdigit(): + print('::error file=%s,line=%s::%s' % (parts[i], parts[i + 1], ':'.join(parts[i + 2:]))) + break + + +def main(args): + root_path = os.path.abspath(args.path) + cmd = args.cmd.split(' ') + if not os.path.exists(root_path): + print('Nonexistent path: %s' % root_path) + sys.exit(2) + err = False + for cur, dirnames, filenames in os.walk(root_path): + parts = cur.replace('\\', '/').split('/') + if '.git' in parts or 'depends' in parts: + continue + for filename in filenames: + if not filename.endswith('.' + args.ext): + continue + full_path = os.path.join(cur, filename) + try: + p = subprocess.Popen(cmd + [full_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + _, stderr = p.communicate() + stderr = stderr.decode('utf-8', errors='ignore') + if stderr: + print_stderr(stderr, args) + if p.returncode != 0: + err = True + except subprocess.CalledProcessError: + err = True + except IOError: + if not err: + print('Warning: cannot check %s script syntax' % args.ext) + err = True + sys.exit(int(err)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--path', default='.', help='Root directory') + parser.add_argument('--ext', help='Script extension', required=True) + parser.add_argument('--cmd', help='Command', required=True) + parser.add_argument('--github-actions', action='store_true', + help='Enable GitHub Actions workflow command output') + args = parser.parse_args() + main(args) diff --git a/package/windows/sdl license.txt b/package/windows/sdl license.txt index e819febfb..4362b4915 100644 --- a/package/windows/sdl license.txt +++ b/package/windows/sdl license.txt @@ -1,5 +1,5 @@ - GNU LESSER GENERAL PUBLIC LICENSE - Version 2.1, February 1999 + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 Copyright (C) 1991, 1999 Free Software Foundation, Inc. 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA @@ -10,7 +10,7 @@ as the successor of the GNU Library Public License, version 2, hence the version number 2.1.] - Preamble + Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public @@ -112,7 +112,7 @@ modification follow. Pay close attention to the difference between a former contains code derived from the library, whereas the latter must be combined with the library in order to run. - GNU LESSER GENERAL PUBLIC LICENSE + GNU LESSER GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License Agreement applies to any software library or other @@ -146,7 +146,7 @@ such a program is covered only if its contents constitute a work based on the Library (independent of the use of the Library in a tool for writing it). Whether that is true depends on what the Library does and what the program that uses the Library does. - + 1. You may copy and distribute verbatim copies of the Library's complete source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an @@ -432,7 +432,7 @@ decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. - NO WARRANTY + NO WARRANTY 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. @@ -455,7 +455,7 @@ FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - END OF TERMS AND CONDITIONS + END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Libraries @@ -499,4 +499,4 @@ necessary. Here is a sample; alter the names: , 1 April 1990 Ty Coon, President of Vice -That's all there is to it! \ No newline at end of file +That's all there is to it! diff --git a/plugins/rendermax/CMakeLists.txt b/plugins/rendermax/CMakeLists.txt index 651fb1e06..349bae13b 100644 --- a/plugins/rendermax/CMakeLists.txt +++ b/plugins/rendermax/CMakeLists.txt @@ -3,12 +3,12 @@ project(rendermax) # A list of source files set(PROJECT_SRCS rendermax.cpp - renderer_light.cpp + renderer_light.cpp ) # A list of headers set(PROJECT_HDRS renderer_opengl.hpp - renderer_light.hpp + renderer_light.hpp ) set_source_files_properties(${PROJECT_HDRS} PROPERTIES HEADER_FILE_ONLY TRUE) diff --git a/plugins/ruby/codegen.pl b/plugins/ruby/codegen.pl index 535a49694..95d91c3ab 100755 --- a/plugins/ruby/codegen.pl +++ b/plugins/ruby/codegen.pl @@ -787,7 +787,7 @@ sub sizeof { } elsif ($subtype eq 'df-linked-list') { return 3 * $SIZEOF_PTR; } elsif ($subtype eq 'df-flagarray') { - return 2 * $SIZEOF_PTR; # XXX length may be 4 on windows? + return 2 * $SIZEOF_PTR; # XXX length may be 4 on windows? } elsif ($subtype eq 'df-static-flagarray') { return $field->getAttribute('count'); } elsif ($subtype eq 'df-array') { diff --git a/plugins/stonesense b/plugins/stonesense index d81cb598e..296ba91c2 160000 --- a/plugins/stonesense +++ b/plugins/stonesense @@ -1 +1 @@ -Subproject commit d81cb598e7aa179bc85440d3ba06ef6ec119815f +Subproject commit 296ba91c2d7e3b011895df17caa032e90a27e186 diff --git a/travis/all.py b/travis/all.py deleted file mode 100755 index edd2eee18..000000000 --- a/travis/all.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python3 -import argparse, os, sys, time - -parser = argparse.ArgumentParser() -parser.add_argument('-n', '--dry-run', action='store_true', help='Display commands without running them') -args = parser.parse_args() - -red = '\x1b[31m\x1b[1m' -green = '\x1b[32m\x1b[1m' -reset = '\x1b(B\x1b[m' -if os.environ.get('TRAVIS', '') == 'true': - print('This script cannot be used in a travis build') - sys.exit(1) -os.chdir(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -commands = [] -with open('.travis.yml') as f: - lines = list(f.readlines()) - script_found = False - for line in lines: - if line.startswith('script:'): - script_found = True - elif script_found: - if line.startswith('- '): - if line.startswith('- python '): - commands.append(line[2:].rstrip('\r\n')) - else: - break -ret = 0 -for cmd in commands: - print('$ %s' % cmd) - if args.dry_run: - continue - start = time.time() - code = os.system(cmd) - end = time.time() - if code != 0: - ret = 1 - print('\n%sThe command "%s" exited with %i.%s [%.3f secs]' % - (green if code == 0 else red, cmd, code, reset, end - start)) - -print('\nDone. Your build exited with %i.' % ret) -sys.exit(ret) diff --git a/travis/authors-rst.py b/travis/authors-rst.py index 076f7f226..cf52385b5 100755 --- a/travis/authors-rst.py +++ b/travis/authors-rst.py @@ -1,70 +1,14 @@ #!/usr/bin/env python3 -""" Overly-complicated script to check formatting/sorting in Authors.rst """ -import os, re, sys +import os +import subprocess +import sys -def main(): - success = [True] - def error(line, msg, **kwargs): - info = '' - for k in kwargs: - info += ' %s %s:' % (k, kwargs[k]) - print('line %i:%s %s' % (line, info, msg)) - if os.environ.get('GITHUB_ACTIONS'): - print('::error file=docs/Authors.rst,line=%i::%s %s' % (line, info.lstrip(), msg)) - success[0] = False - with open('docs/Authors.rst', 'rb') as f: - lines = list(map(lambda line: line.decode('utf8').replace('\n', ''), f.readlines())) +script_name = os.path.basename(__file__) +new_script_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'ci', script_name) - if lines[1].startswith('='): - if len(lines[0]) != len(lines[1]): - error(2, 'Length of header does not match underline') - if lines[1].replace('=', ''): - error(2, 'Invalid header') +sys.stderr.write('\nNote: travis/{script_name} is deprecated. Use ci/{script_name} instead.\n\n'.format(script_name=script_name)) +sys.stderr.flush() - first_div_index = list(filter(lambda pair: pair[1].startswith('==='), enumerate(lines[2:])))[0][0] + 2 - first_div = lines[first_div_index] - div_indices = [] - for i, line in enumerate(lines[first_div_index:]): - line_number = i + first_div_index + 1 - if '\t' in line: - error(line_number, 'contains tabs') - if line.startswith('==='): - div_indices.append(i + first_div_index) - if not re.match(r'^=+( =+)+$', line): - error(line_number, 'bad table divider') - if line != lines[first_div_index]: - error(line_number, 'malformed table divider') - if len(div_indices) < 3: - error(len(lines), 'missing table divider(s)') - for i in div_indices[3:]: - error(i + 1, 'extra table divider') - - col_ranges = [] - i = 0 - while True: - j = first_div.find(' ', i) - col_ranges.append(slice(i, j if j > 0 else None)) - if j == -1: - break - i = j + 1 - - for i, line in enumerate(lines[div_indices[1] + 1:div_indices[2]]): - line_number = i + div_indices[1] + 2 - for c, col in enumerate(col_ranges): - cell = line[col] - if cell.startswith(' '): - error(line_number, 'text does not start in correct location', column=c+1) - # check for text extending into next column if this isn't the last column - if col.stop is not None and col.stop < len(line) and line[col.stop] != ' ': - error(line_number, 'text extends into next column', column=c+1) - if i > 0: - prev_line = lines[div_indices[1] + i] - if line.lower()[col_ranges[0]] < prev_line.lower()[col_ranges[0]]: - error(line_number, 'not sorted: should come before line %i ("%s" before "%s")' % - (line_number - 1, line[col_ranges[0]].rstrip(' '), prev_line[col_ranges[0]].rstrip(' '))) - - return success[0] - -if __name__ == '__main__': - sys.exit(int(not main())) +p = subprocess.run([sys.executable, new_script_path] + sys.argv[1:]) +sys.exit(p.returncode) diff --git a/travis/buildmaster-rebuild-pr.py b/travis/buildmaster-rebuild-pr.py new file mode 100755 index 000000000..cf52385b5 --- /dev/null +++ b/travis/buildmaster-rebuild-pr.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 + +import os +import subprocess +import sys + +script_name = os.path.basename(__file__) +new_script_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'ci', script_name) + +sys.stderr.write('\nNote: travis/{script_name} is deprecated. Use ci/{script_name} instead.\n\n'.format(script_name=script_name)) +sys.stderr.flush() + +p = subprocess.run([sys.executable, new_script_path] + sys.argv[1:]) +sys.exit(p.returncode) diff --git a/travis/check-rpc.py b/travis/check-rpc.py index aba3e3811..cf52385b5 100755 --- a/travis/check-rpc.py +++ b/travis/check-rpc.py @@ -1,110 +1,14 @@ #!/usr/bin/env python3 -import glob -import sys - -actual = {'': {}} - -with open(sys.argv[1]) as f: - plugin_name = '' - for line in f: - line = line.rstrip() - if line.startswith('// Plugin: '): - plugin_name = line.split(' ')[2] - if plugin_name not in actual: - actual[plugin_name] = {} - elif line.startswith('// RPC '): - parts = line.split(' ') - actual[plugin_name][parts[2]] = (parts[4], parts[6]) - -expected = {'': {}} - -for p in glob.iglob('library/proto/*.proto'): - with open(p) as f: - for line in f: - line = line.rstrip() - if line.startswith('// RPC '): - parts = line.split(' ') - expected[''][parts[2]] = (parts[4], parts[6]) - -for p in glob.iglob('plugins/proto/*.proto'): - plugin_name = '' - with open(p) as f: - for line in f: - line = line.rstrip() - if line.startswith('// Plugin: '): - plugin_name = line.split(' ')[2] - if plugin_name not in expected: - expected[plugin_name] = {} - break - - if plugin_name == '': - continue - with open(p) as f: - for line in f: - line = line.rstrip() - if line.startswith('// RPC '): - parts = line.split(' ') - expected[plugin_name][parts[2]] = (parts[4], parts[6]) - -error_count = 0 - -for plugin_name in actual: - methods = actual[plugin_name] - - if plugin_name not in expected: - print('Missing documentation for plugin proto files: ' + plugin_name) - print('Add the following lines:') - print('// Plugin: ' + plugin_name) - error_count += 1 - for m in methods: - io = methods[m] - print('// RPC ' + m + ' : ' + io[0] + ' -> ' + io[1]) - error_count += 1 - else: - missing = [] - wrong = [] - for m in methods: - io = methods[m] - if m in expected[plugin_name]: - if expected[plugin_name][m] != io: - wrong.append('// RPC ' + m + ' : ' + io[0] + ' -> ' + io[1]) - else: - missing.append('// RPC ' + m + ' : ' + io[0] + ' -> ' + io[1]) - - if len(missing) > 0: - print('Incomplete documentation for ' + ('core' if plugin_name == '' else 'plugin "' + plugin_name + '"') + ' proto files. Add the following lines:') - for m in missing: - print(m) - error_count += 1 - - if len(wrong) > 0: - print('Incorrect documentation for ' + ('core' if plugin_name == '' else 'plugin "' + plugin_name + '"') + ' proto files. Replace the following comments:') - for m in wrong: - print(m) - error_count += 1 - -for plugin_name in expected: - methods = expected[plugin_name] +import os +import subprocess +import sys - if plugin_name not in actual: - print('Incorrect documentation for plugin proto files: ' + plugin_name) - print('The following methods are documented, but the plugin does not provide any RPC methods:') - for m in methods: - io = methods[m] - print('// RPC ' + m + ' : ' + io[0] + ' -> ' + io[1]) - error_count += 1 - else: - missing = [] - for m in methods: - io = methods[m] - if m not in actual[plugin_name]: - missing.append('// RPC ' + m + ' : ' + io[0] + ' -> ' + io[1]) +script_name = os.path.basename(__file__) +new_script_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'ci', script_name) - if len(missing) > 0: - print('Incorrect documentation for ' + ('core' if plugin_name == '' else 'plugin "' + plugin_name + '"') + ' proto files. Remove the following lines:') - for m in missing: - print(m) - error_count += 1 +sys.stderr.write('\nNote: travis/{script_name} is deprecated. Use ci/{script_name} instead.\n\n'.format(script_name=script_name)) +sys.stderr.flush() -sys.exit(min(100, error_count)) +p = subprocess.run([sys.executable, new_script_path] + sys.argv[1:]) +sys.exit(p.returncode) diff --git a/travis/download-df.sh b/travis/download-df.sh index 49dcedea7..aec2d6d99 100755 --- a/travis/download-df.sh +++ b/travis/download-df.sh @@ -1,56 +1,9 @@ #!/bin/sh -set -e +script_name="$(basename "$0")" +new_script_path="$(dirname "$0")/../ci/${script_name}" -tardest="df.tar.bz2" +printf >&2 "\nNote: travis/%s is deprecated. Use ci/%s instead.\n\n" "${script_name}" "${script_name}" -selfmd5=$(openssl md5 < "$0") -echo $selfmd5 - -cd "$(dirname "$0")" -echo "DF_VERSION: $DF_VERSION" -echo "DF_FOLDER: $DF_FOLDER" -mkdir -p "$DF_FOLDER" -# back out of df_linux -cd "$DF_FOLDER/.." - -if [ -f receipt ]; then - if [ "$selfmd5" != "$(cat receipt)" ]; then - echo "download-df.sh changed; removing DF" - rm receipt - else - echo "Already downloaded $DF_VERSION" - fi -fi - -if [ ! -f receipt ]; then - rm -f "$tardest" - minor=$(echo "$DF_VERSION" | cut -d. -f2) - patch=$(echo "$DF_VERSION" | cut -d. -f3) - url="http://www.bay12games.com/dwarves/df_${minor}_${patch}_linux.tar.bz2" - echo Downloading - while read url; do - echo "Attempting download: ${url}" - if wget -v "$url" -O "$tardest"; then - break - fi - done < receipt -ls +"${new_script_path}" "$@" +exit $? diff --git a/travis/get-df-version.sh b/travis/get-df-version.sh index 24386252d..aec2d6d99 100755 --- a/travis/get-df-version.sh +++ b/travis/get-df-version.sh @@ -1,4 +1,9 @@ #!/bin/sh -cd "$(dirname "$0")" -cd .. -grep -i 'set(DF_VERSION' CMakeLists.txt | perl -ne 'print "$&\n" if /[\d\.]+/' + +script_name="$(basename "$0")" +new_script_path="$(dirname "$0")/../ci/${script_name}" + +printf >&2 "\nNote: travis/%s is deprecated. Use ci/%s instead.\n\n" "${script_name}" "${script_name}" + +"${new_script_path}" "$@" +exit $? diff --git a/travis/lint.py b/travis/lint.py index 05aa28950..cf52385b5 100755 --- a/travis/lint.py +++ b/travis/lint.py @@ -1,155 +1,14 @@ #!/usr/bin/env python3 -import re, os, sys -valid_extensions = ['c', 'cpp', 'h', 'hpp', 'mm', 'lua', 'rb', 'proto', - 'init', 'init-example', 'rst'] -path_blacklist = [ - '^library/include/df/', - '^plugins/stonesense/allegro', - '^plugins/isoworld/allegro', - '^plugins/isoworld/agui', - '^depends/', - '^.git/', - '^build', - '.pb.h', -] +import os +import subprocess +import sys -def valid_file(filename): - return len(list(filter(lambda ext: filename.endswith('.' + ext), valid_extensions))) and \ - not len(list(filter(lambda path: path.replace('\\', '/') in filename.replace('\\', '/'), path_blacklist))) +script_name = os.path.basename(__file__) +new_script_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'ci', script_name) -success = True -def error(msg=None): - global success - success = False - if msg: - sys.stderr.write(msg + '\n') +sys.stderr.write('\nNote: travis/{script_name} is deprecated. Use ci/{script_name} instead.\n\n'.format(script_name=script_name)) +sys.stderr.flush() -def format_lines(lines, total): - if len(lines) == total - 1: - return 'entire file' - if not len(lines): - # should never happen - return 'nowhere' - if len(lines) == 1: - return 'line %i' % lines[0] - s = 'lines ' - range_start = range_end = lines[0] - for i, line in enumerate(lines): - if line > range_end + 1: - if range_start == range_end: - s += ('%i, ' % range_end) - else: - s += ('%i-%i, ' % (range_start, range_end)) - range_start = range_end = line - if i == len(lines) - 1: - s += ('%i' % line) - else: - range_end = line - if i == len(lines) - 1: - s += ('%i-%i, ' % (range_start, range_end)) - return s.rstrip(' ').rstrip(',') - -class LinterError(Exception): - def __init__(self, message, lines, total_lines): - self.message = message - self.lines = lines - self.total_lines = total_lines - - def __str__(self): - return '%s: %s' % (self.message, format_lines(self.lines, self.total_lines)) - - def github_actions_workflow_command(self, filename): - first_line = self.lines[0] if self.lines else 1 - return '::error file=%s,line=%i::%s' % (filename, first_line, self) - -class Linter(object): - ignore = False - def check(self, lines): - failures = [] - for i, line in enumerate(lines): - if not self.check_line(line): - failures.append(i + 1) - if len(failures): - raise LinterError(self.msg, failures, len(lines)) - - def fix(self, lines): - for i in range(len(lines)): - lines[i] = self.fix_line(lines[i]) - - -class NewlineLinter(Linter): - msg = 'Contains DOS-style newlines' - # git supports newline conversion. Catch in CI, ignore on Windows. - ignore = os.linesep != '\n' and not os.environ.get('TRAVIS') - def check_line(self, line): - return '\r' not in line - def fix_line(self, line): - return line.replace('\r', '') - -class TrailingWhitespaceLinter(Linter): - msg = 'Contains trailing whitespace' - def check_line(self, line): - line = line.replace('\r', '').replace('\n', '') - return not line.strip() or line == line.rstrip('\t ') - def fix_line(self, line): - return line.rstrip('\t ') - -class TabLinter(Linter): - msg = 'Contains tabs' - def check_line(self, line): - return '\t' not in line - def fix_line(self, line): - return line.replace('\t', ' ') - -linters = [cls() for cls in Linter.__subclasses__() if not cls.ignore] - -def main(): - is_github_actions = bool(os.environ.get('GITHUB_ACTIONS')) - root_path = os.path.abspath(sys.argv[1] if len(sys.argv) > 1 else '.') - if not os.path.exists(root_path): - print('Nonexistent path: %s' % root_path) - sys.exit(2) - fix = (len(sys.argv) > 2 and sys.argv[2] == '--fix') - global path_blacklist - path_blacklist = list(map(lambda s: os.path.join(root_path, s.replace('^', '')) if s.startswith('^') else s, path_blacklist)) - - for cur, dirnames, filenames in os.walk(root_path): - for filename in filenames: - full_path = os.path.join(cur, filename) - rel_path = full_path.replace(root_path, '.') - if not valid_file(full_path): - continue - lines = [] - with open(full_path, 'rb') as f: - lines = f.read().split(b'\n') - for i, line in enumerate(lines): - try: - lines[i] = line.decode('utf-8') - except UnicodeDecodeError: - msg_params = (rel_path, i + 1, 'Invalid UTF-8 (other errors will be ignored)') - error('%s:%i: %s' % msg_params) - if is_github_actions: - print('::error file=%s,line=%i::%s' % msg_params) - lines[i] = '' - for linter in linters: - try: - linter.check(lines) - except LinterError as e: - error('%s: %s' % (rel_path, e)) - if is_github_actions: - print(e.github_actions_workflow_command(rel_path)) - if fix: - linter.fix(lines) - contents = '\n'.join(lines) - with open(full_path, 'wb') as f: - f.write(contents) - - if success: - print('All linters completed successfully') - sys.exit(0) - else: - sys.exit(1) - -if __name__ == '__main__': - main() +p = subprocess.run([sys.executable, new_script_path] + sys.argv[1:]) +sys.exit(p.returncode) diff --git a/travis/run-tests.py b/travis/run-tests.py index 132631903..cf52385b5 100755 --- a/travis/run-tests.py +++ b/travis/run-tests.py @@ -1,127 +1,14 @@ #!/usr/bin/env python3 -import argparse -import enum -import json + import os -import re -import shutil import subprocess import sys -parser = argparse.ArgumentParser() -parser.add_argument('df_folder', help='DF base folder') -parser.add_argument('--headless', action='store_true', - help='Run without opening DF window (requires non-Windows)') -parser.add_argument('--keep-status', action='store_true', - help='Do not delete final status file') -parser.add_argument('--no-quit', action='store_true', - help='Do not quit DF when done') -parser.add_argument('--test-dir', '--test-folder', - help='Base test folder (default: df_folder/test)') -parser.add_argument('-t', '--test', dest='tests', nargs='+', - help='Test(s) to run (Lua patterns accepted)') -args = parser.parse_args() - -if (not sys.stdin.isatty() or not sys.stdout.isatty() or not sys.stderr.isatty()) and not args.headless: - print('WARN: no TTY detected, enabling headless mode') - args.headless = True - -if args.test_dir is not None: - args.test_dir = os.path.normpath(os.path.join(os.getcwd(), args.test_dir)) - if not os.path.isdir(args.test_dir): - print('ERROR: invalid test folder: %r' % args.test_dir) - -MAX_TRIES = 5 - -dfhack = 'Dwarf Fortress.exe' if sys.platform == 'win32' else './dfhack' -test_status_file = 'test_status.json' - -class TestStatus(enum.Enum): - PENDING = 'pending' - PASSED = 'passed' - FAILED = 'failed' - -def get_test_status(): - if os.path.isfile(test_status_file): - with open(test_status_file) as f: - return {k: TestStatus(v) for k, v in json.load(f).items()} - -def change_setting(content, setting, value): - return '[' + setting + ':' + value + ']\n' + re.sub( - r'\[' + setting + r':.+?\]', '(overridden)', content, flags=re.IGNORECASE) - -os.chdir(args.df_folder) -if os.path.exists(test_status_file): - os.remove(test_status_file) - -print('Backing up init.txt to init.txt.orig') -init_txt_path = 'data/init/init.txt' -shutil.copyfile(init_txt_path, init_txt_path + '.orig') -with open(init_txt_path) as f: - init_contents = f.read() -init_contents = change_setting(init_contents, 'INTRO', 'NO') -init_contents = change_setting(init_contents, 'SOUND', 'NO') -init_contents = change_setting(init_contents, 'WINDOWED', 'YES') -init_contents = change_setting(init_contents, 'WINDOWEDX', '80') -init_contents = change_setting(init_contents, 'WINDOWEDY', '25') -init_contents = change_setting(init_contents, 'FPS', 'YES') -if args.headless: - init_contents = change_setting(init_contents, 'PRINT_MODE', 'TEXT') - -test_init_file = 'dfhackzzz_test.init' # Core sorts these alphabetically -with open(test_init_file, 'w') as f: - f.write(''' - devel/dump-rpc dfhack-rpc.txt - :lua dfhack.internal.addScriptPath(dfhack.getHackPath()) - test --resume --modes=none,title "lua scr.breakdown_level=df.interface_breakdown_types.%s" - ''' % ('NONE' if args.no_quit else 'QUIT')) - -test_config_file = 'test_config.json' -with open(test_config_file, 'w') as f: - json.dump({ - 'test_dir': args.test_dir, - 'tests': args.tests, - }, f) - -try: - with open(init_txt_path, 'w') as f: - f.write(init_contents) - - tries = 0 - while True: - status = get_test_status() - if status is not None: - if all(s != TestStatus.PENDING for s in status.values()): - print('Done!') - sys.exit(int(any(s != TestStatus.PASSED for s in status.values()))) - elif tries > 0: - print('ERROR: Could not read status file') - sys.exit(2) - - tries += 1 - print('Starting DF: #%i' % (tries)) - if tries > MAX_TRIES: - print('ERROR: Too many tries - aborting') - sys.exit(1) +script_name = os.path.basename(__file__) +new_script_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'ci', script_name) - if args.headless: - os.environ['DFHACK_HEADLESS'] = '1' - os.environ['DFHACK_DISABLE_CONSOLE'] = '1' +sys.stderr.write('\nNote: travis/{script_name} is deprecated. Use ci/{script_name} instead.\n\n'.format(script_name=script_name)) +sys.stderr.flush() - process = subprocess.Popen([dfhack], - stdin=subprocess.PIPE if args.headless else sys.stdin, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - _, err = process.communicate() - if err: - print('WARN: DF produced stderr: ' + repr(err[:5000])) - if process.returncode != 0: - print('ERROR: DF exited with ' + repr(process.returncode)) -finally: - print('\nRestoring original init.txt') - shutil.copyfile(init_txt_path + '.orig', init_txt_path) - if os.path.isfile(test_init_file): - os.remove(test_init_file) - if not args.keep_status and os.path.isfile(test_status_file): - os.remove(test_status_file) - print('Cleanup done') +p = subprocess.run([sys.executable, new_script_path] + sys.argv[1:]) +sys.exit(p.returncode) diff --git a/travis/script-docs.py b/travis/script-docs.py index 71d7f37b2..cf52385b5 100755 --- a/travis/script-docs.py +++ b/travis/script-docs.py @@ -1,94 +1,14 @@ #!/usr/bin/env python3 + import os -from os.path import basename, dirname, join, splitext +import subprocess import sys -SCRIPT_PATH = sys.argv[1] if len(sys.argv) > 1 else 'scripts' -IS_GITHUB_ACTIONS = bool(os.environ.get('GITHUB_ACTIONS')) - -def expected_cmd(path): - """Get the command from the name of a script.""" - dname, fname = basename(dirname(path)), splitext(basename(path))[0] - if dname in ('devel', 'fix', 'gui', 'modtools'): - return dname + '/' + fname - return fname - - -def check_ls(fname, line): - """Check length & existence of leading comment for "ls" builtin command.""" - line = line.strip() - comment = '--' if fname.endswith('.lua') else '#' - if '[====[' in line or not line.startswith(comment): - print_error('missing leading comment (requred for `ls`)', fname) - return 1 - return 0 - - -def print_error(message, filename, line=None): - if not isinstance(line, int): - line = 1 - print('Error: %s:%i: %s' % (filename, line, message)) - if IS_GITHUB_ACTIONS: - print('::error file=%s,line=%i::%s' % (filename, line, message)) - - -def check_file(fname): - errors, doclines = 0, [] - tok1, tok2 = ('=begin', '=end') if fname.endswith('.rb') else \ - ('[====[', ']====]') - doc_start_line = None - with open(fname, errors='ignore') as f: - lines = f.readlines() - if not lines: - print_error('empty file', fname) - return 1 - errors += check_ls(fname, lines[0]) - for i, l in enumerate(lines): - if doclines or l.strip().endswith(tok1): - if not doclines: - doc_start_line = i + 1 - doclines.append(l.rstrip()) - if l.startswith(tok2): - break - else: - if doclines: - print_error('docs start but do not end', fname, doc_start_line) - else: - print_error('no documentation found', fname) - return 1 - - if not doclines: - print_error('missing or malformed documentation', fname) - return 1 - - title, underline = [d for d in doclines - if d and '=begin' not in d and '[====[' not in d][:2] - title_line = doc_start_line + doclines.index(title) - expected_underline = '=' * len(title) - if underline != expected_underline: - print_error('title/underline mismatch: expected {!r}, got {!r}'.format( - expected_underline, underline), - fname, title_line + 1) - errors += 1 - if title != expected_cmd(fname): - print_error('expected script title {!r}, got {!r}'.format( - expected_cmd(fname), title), - fname, title_line) - errors += 1 - return errors - - -def main(): - """Check that all DFHack scripts include documentation""" - err = 0 - exclude = set(['internal', 'test']) - for root, dirs, files in os.walk(SCRIPT_PATH, topdown=True): - dirs[:] = [d for d in dirs if d not in exclude] - for f in files: - if f[-3:] in {'.rb', 'lua'}: - err += check_file(join(root, f)) - return err +script_name = os.path.basename(__file__) +new_script_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'ci', script_name) +sys.stderr.write('\nNote: travis/{script_name} is deprecated. Use ci/{script_name} instead.\n\n'.format(script_name=script_name)) +sys.stderr.flush() -if __name__ == '__main__': - sys.exit(min(100, main())) +p = subprocess.run([sys.executable, new_script_path] + sys.argv[1:]) +sys.exit(p.returncode) diff --git a/travis/script-syntax.py b/travis/script-syntax.py index 593a25a24..cf52385b5 100755 --- a/travis/script-syntax.py +++ b/travis/script-syntax.py @@ -1,64 +1,14 @@ #!/usr/bin/env python3 -import argparse + import os import subprocess import sys +script_name = os.path.basename(__file__) +new_script_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'ci', script_name) -def print_stderr(stderr, args): - if not args.github_actions: - sys.stderr.write(stderr + '\n') - return - - for line in stderr.split('\n'): - print(line) - parts = list(map(str.strip, line.split(':'))) - # e.g. luac prints "luac:" in front of messages, so find the first part - # containing the actual filename - for i in range(len(parts) - 1): - if parts[i].endswith('.' + args.ext) and parts[i + 1].isdigit(): - print('::error file=%s,line=%s::%s' % (parts[i], parts[i + 1], ':'.join(parts[i + 2:]))) - break - - -def main(args): - root_path = os.path.abspath(args.path) - cmd = args.cmd.split(' ') - if not os.path.exists(root_path): - print('Nonexistent path: %s' % root_path) - sys.exit(2) - err = False - for cur, dirnames, filenames in os.walk(root_path): - parts = cur.replace('\\', '/').split('/') - if '.git' in parts or 'depends' in parts: - continue - for filename in filenames: - if not filename.endswith('.' + args.ext): - continue - full_path = os.path.join(cur, filename) - try: - p = subprocess.Popen(cmd + [full_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - _, stderr = p.communicate() - stderr = stderr.decode('utf-8', errors='ignore') - if stderr: - print_stderr(stderr, args) - if p.returncode != 0: - err = True - except subprocess.CalledProcessError: - err = True - except IOError: - if not err: - print('Warning: cannot check %s script syntax' % args.ext) - err = True - sys.exit(int(err)) - +sys.stderr.write('\nNote: travis/{script_name} is deprecated. Use ci/{script_name} instead.\n\n'.format(script_name=script_name)) +sys.stderr.flush() -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--path', default='.', help='Root directory') - parser.add_argument('--ext', help='Script extension', required=True) - parser.add_argument('--cmd', help='Command', required=True) - parser.add_argument('--github-actions', action='store_true', - help='Enable GitHub Actions workflow command output') - args = parser.parse_args() - main(args) +p = subprocess.run([sys.executable, new_script_path] + sys.argv[1:]) +sys.exit(p.returncode)