From ea3be02c6384e6c7ad7412ed301e5e18f26fce10 Mon Sep 17 00:00:00 2001 From: lethosor Date: Wed, 25 Mar 2020 01:43:09 -0400 Subject: [PATCH] Overhaul lua testing script - Now keeps track of the state of each test individually - Only runs uncompleted tests if DF crashes/restarts - DF now exits with 0 - Easier to run locally - Hopefully works on Travis too! --- .travis.yml | 8 +-- test/main.lua | 94 ++++++++++++++++++--------- travis/dfhack_travis.init | 4 -- travis/download-df.sh | 4 -- travis/run-tests.py | 130 ++++++++++++++++++++++++++++---------- 5 files changed, 163 insertions(+), 77 deletions(-) delete mode 100644 travis/dfhack_travis.init diff --git a/.travis.yml b/.travis.yml index 98534a11a..c096552bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,8 +32,6 @@ before_install: - pip install --user "sphinx==1.4" "requests[security]" - sh travis/build-lua.sh - sh travis/download-df.sh -- echo "export DFHACK_HEADLESS=1" >> "$HOME/.dfhackrc" -- echo "export DFHACK_DISABLE_CONSOLE=1" >> "$HOME/.dfhackrc" script: - export PATH="$PATH:$HOME/lua53/bin" - git tag tmp-travis-build @@ -47,13 +45,13 @@ script: - python travis/script-syntax.py --ext=rb --cmd="ruby -c" - mkdir build-travis - cd build-travis -- cmake .. -G Ninja -DCMAKE_C_COMPILER=gcc-$GCC_VERSION -DCMAKE_CXX_COMPILER=g++-$GCC_VERSION -DDFHACK_BUILD_ARCH=64 -DBUILD_DOCS:BOOL=ON -DCMAKE_INSTALL_PREFIX="$DF_FOLDER" +- cmake .. -G Ninja -DCMAKE_C_COMPILER=gcc-$GCC_VERSION -DCMAKE_CXX_COMPILER=g++-$GCC_VERSION -DDFHACK_BUILD_ARCH=64 -DBUILD_DOCS:BOOL=ON -DBUILD_TESTS:BOOL=ON -DCMAKE_INSTALL_PREFIX="$DF_FOLDER" - ninja -j3 install - mv "$DF_FOLDER"/dfhack.init-example "$DF_FOLDER"/dfhack.init - cd .. -- cp travis/dfhack_travis.init "$DF_FOLDER"/ -- python travis/run-tests.py "$DF_FOLDER" +- python travis/run-tests.py --headless --keep-status "$DF_FOLDER" - python travis/check-rpc.py "$DF_FOLDER/dfhack-rpc.txt" +- cat "$DF_FOLDER/test_status.json" before_cache: - cat "$DF_FOLDER/stderr.log" - rm -rf "$DF_FOLDER" diff --git a/test/main.lua b/test/main.lua index 5e9d09302..04313aea2 100644 --- a/test/main.lua +++ b/test/main.lua @@ -1,9 +1,17 @@ +local json = require 'json' local script = require 'gui.script' local utils = require 'utils' local args = {...} local done_command = args[1] +local STATUS_FILE = 'test_status.json' +local TestStatus = { + PENDING = 'pending', + PASSED = 'passed', + FAILED = 'failed', +} + expect = {} function expect.true_(value, comment) return not not value, comment, 'expected true' @@ -78,18 +86,17 @@ function get_test_files() return files end -function set_test_stage(stage) - local f = io.open('test_stage.txt', 'w') - f:write(stage) - f:close() +function load_test_status() + if dfhack.filesystem.isfile(STATUS_FILE) then + return json.decode_file(STATUS_FILE) + end end -function finish_tests(ok) - if ok then - print('Tests finished') - else - dfhack.printerr('Tests failed!') - end +function save_test_status(status) + json.encode_file(status, STATUS_FILE) +end + +function finish_tests() if done_command then dfhack.run_command(done_command) end @@ -122,9 +129,11 @@ function main() qerror('Could not find title screen') end - print('Running tests') + print('Loading tests') + local tests = {} for _, file in ipairs(files) do - print('Running file: ' .. file:sub(file:find('test'), -1)) + local short_filename = file:sub(file:find('test'), -1) + print('Loading file: ' .. short_filename) local env, env_private = build_test_env() local code, err = loadfile(file, 't', env) if not code then @@ -138,35 +147,58 @@ function main() counts.file_errors = counts.file_errors + 1 dfhack.printerr('Error when running file: ' .. tostring(err)) else - for name, test in pairs(env.test) do - env_private.checks = 0 - env_private.checks_ok = 0 - counts.tests = counts.tests + 1 - local ok, err = pcall(test) - if not ok then - dfhack.printerr('test errored: ' .. name .. ': ' .. tostring(err)) - passed = false - elseif env_private.checks ~= env_private.checks_ok then - dfhack.printerr('test failed: ' .. name) - passed = false - else - print('test passed: ' .. name) - counts.tests_ok = counts.tests_ok + 1 - end - counts.checks = counts.checks + (tonumber(env_private.checks) or 0) - counts.checks_ok = counts.checks_ok + (tonumber(env_private.checks_ok) or 0) + for name, test_func in pairs(env.test) do + local test_data = { + full_name = short_filename .. ':' .. name, + func = test_func, + private = env_private, + } + test_data.name = test_data.full_name:gsub('test/', ''):gsub('.lua', '') + table.insert(tests, test_data) end end end end + local status = load_test_status() or {} + for _, test in pairs(tests) do + if not status[test.full_name] then + status[test.full_name] = TestStatus.PENDING + end + end + + print('Running tests') + for _, test in pairs(tests) do + if status[test.full_name] == TestStatus.PENDING then + test.private.checks = 0 + test.private.checks_ok = 0 + counts.tests = counts.tests + 1 + local ok, err = pcall(test.func) + local passed = false + if not ok then + dfhack.printerr('test errored: ' .. test.name .. ': ' .. tostring(err)) + elseif test.private.checks ~= test.private.checks_ok then + dfhack.printerr('test failed: ' .. test.name) + else + print('test passed: ' .. test.name) + passed = true + counts.tests_ok = counts.tests_ok + 1 + end + counts.checks = counts.checks + (tonumber(test.private.checks) or 0) + counts.checks_ok = counts.checks_ok + (tonumber(test.private.checks_ok) or 0) + status[test.full_name] = passed and TestStatus.PASSED or TestStatus.FAILED + save_test_status(status) + else + print('test skipped: ' .. test.name .. ' (state = ' .. status[test.full_name] .. ')') + end + end + print('\nTest summary:') print(('%d/%d tests passed'):format(counts.tests_ok, counts.tests)) print(('%d/%d checks passed'):format(counts.checks_ok, counts.checks)) print(('%d test files failed to load'):format(counts.file_errors)) - set_test_stage(passed and 'done' or 'fail') - finish_tests(passed) + finish_tests() end script.start(main) diff --git a/travis/dfhack_travis.init b/travis/dfhack_travis.init deleted file mode 100644 index 9c74dc3b4..000000000 --- a/travis/dfhack_travis.init +++ /dev/null @@ -1,4 +0,0 @@ -devel/dump-rpc dfhack-rpc.txt - -:lua dfhack.internal.addScriptPath(dfhack.getHackPath()) -test/main die diff --git a/travis/download-df.sh b/travis/download-df.sh index 4eae6f9b9..834aa4da7 100755 --- a/travis/download-df.sh +++ b/travis/download-df.sh @@ -36,10 +36,6 @@ mkdir df_linux echo Extracting tar xf "$tardest" --strip-components=1 -C df_linux -echo Changing settings -echo '' >> "df_linux/data/init/init.txt" -echo '[PRINT_MODE:TEXT]' >> "df_linux/data/init/init.txt" -echo '[SOUND:NO]' >> "df_linux/data/init/init.txt" echo Done echo "$selfmd5" > receipt diff --git a/travis/run-tests.py b/travis/run-tests.py index a66320f37..eec771e9b 100644 --- a/travis/run-tests.py +++ b/travis/run-tests.py @@ -1,40 +1,104 @@ -import os, subprocess, sys +import argparse +import enum +import json +import os +import re +import shutil +import subprocess +import sys + +parser = argparse.ArgumentParser() +parser.add_argument('df_folder', help='DF base folder') +parser.add_argument('--headless', action='store_true', + help='Run without opening DF window (requires non-Windows)') +parser.add_argument('--keep-status', action='store_true', + help='Do not delete final status file') +args = parser.parse_args() MAX_TRIES = 5 dfhack = 'Dwarf Fortress.exe' if sys.platform == 'win32' else './dfhack' -test_stage = 'test_stage.txt' +test_status_file = 'test_status.json' + +class TestStatus(enum.Enum): + PENDING = 'pending' + PASSED = 'passed' + FAILED = 'failed' + +def get_test_status(): + if os.path.isfile(test_status_file): + with open(test_status_file) as f: + return {k: TestStatus(v) for k, v in json.load(f).items()} -def get_test_stage(): - if os.path.isfile(test_stage): - return open(test_stage).read().strip() - return '0' +def change_setting(content, setting, value): + return '[' + setting + ':' + value + ']\n' + re.sub( + r'\[' + setting + r':.+?\]', '(overridden)', content, flags=re.IGNORECASE) os.chdir(sys.argv[1]) -if os.path.exists(test_stage): - os.remove(test_stage) - -print(os.getcwd()) -print(os.listdir('.')) - -tries = 0 -while True: - tries += 1 - stage = get_test_stage() - print('Run #%i: stage=%s' % (tries, get_test_stage())) - if stage == 'done': - print('Done!') - os.remove(test_stage) - sys.exit(0) - elif stage == 'fail': - print('Failed!') - os.remove(test_stage) - sys.exit(1) - if tries > MAX_TRIES: - print('Too many tries - aborting') - sys.exit(1) - - process = subprocess.Popen([dfhack], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) - process.communicate() - if process.returncode != 0: - print('DF exited with ' + repr(process.returncode)) +if os.path.exists(test_status_file): + os.remove(test_status_file) + +print('Backing up init.txt to init.txt.orig') +init_txt_path = 'data/init/init.txt' +shutil.copyfile(init_txt_path, init_txt_path + '.orig') +with open(init_txt_path) as f: + init_contents = f.read() +init_contents = change_setting(init_contents, 'INTRO', 'NO') +init_contents = change_setting(init_contents, 'SOUND', 'NO') +init_contents = change_setting(init_contents, 'WINDOWED', 'YES') +init_contents = change_setting(init_contents, 'WINDOWEDX', '80') +init_contents = change_setting(init_contents, 'WINDOWEDY', '25') +init_contents = change_setting(init_contents, 'FPS', 'YES') +if args.headless: + init_contents = change_setting(init_contents, 'PRINT_MODE', 'TEXT') + +test_init_file = 'dfhackzzz_test.init' # Core sorts these alphabetically +with open(test_init_file, 'w') as f: + f.write(''' + devel/dump-rpc dfhack-rpc.txt + :lua dfhack.internal.addScriptPath(dfhack.getHackPath()) + test/main "lua scr.breakdown_level=df.interface_breakdown_types.QUIT" + ''') + +try: + with open(init_txt_path, 'w') as f: + f.write(init_contents) + + tries = 0 + while True: + status = get_test_status() + if status is not None: + if all(s is not TestStatus.PENDING for s in status.values()): + print('Done!') + break + elif tries > 0: + print('ERROR: Could not read status file') + sys.exit(2) + + tries += 1 + print('Starting DF: #%i' % (tries)) + if tries > MAX_TRIES: + print('ERROR: Too many tries - aborting') + sys.exit(1) + + if args.headless: + os.environ['DFHACK_HEADLESS'] = '1' + os.environ['DFHACK_DISABLE_CONSOLE'] = '1' + + process = subprocess.Popen([dfhack], + stdin=subprocess.PIPE if args.headless else sys.stdin, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + _, err = process.communicate() + if err: + print('WARN: DF produced stderr: ' + repr(err[:5000])) + if process.returncode != 0: + print('WARN: DF exited with ' + repr(process.returncode)) +finally: + print('\nRestoring original init.txt') + shutil.copyfile(init_txt_path + '.orig', init_txt_path) + if os.path.isfile(test_init_file): + os.remove(test_init_file) + if not args.keep_status and os.path.isfile(test_status_file): + os.remove(test_status_file) + print('Cleanup done')