Overhaul lua testing script

- Now keeps track of the state of each test individually
- Only runs uncompleted tests if DF crashes/restarts
- DF now exits with 0
- Easier to run locally
- Hopefully works on Travis too!
develop
lethosor 2020-03-25 01:43:09 -04:00
parent 90d1f26504
commit ea3be02c63
5 changed files with 163 additions and 77 deletions

@ -32,8 +32,6 @@ before_install:
- pip install --user "sphinx==1.4" "requests[security]" - pip install --user "sphinx==1.4" "requests[security]"
- sh travis/build-lua.sh - sh travis/build-lua.sh
- sh travis/download-df.sh - sh travis/download-df.sh
- echo "export DFHACK_HEADLESS=1" >> "$HOME/.dfhackrc"
- echo "export DFHACK_DISABLE_CONSOLE=1" >> "$HOME/.dfhackrc"
script: script:
- export PATH="$PATH:$HOME/lua53/bin" - export PATH="$PATH:$HOME/lua53/bin"
- git tag tmp-travis-build - git tag tmp-travis-build
@ -47,13 +45,13 @@ script:
- python travis/script-syntax.py --ext=rb --cmd="ruby -c" - python travis/script-syntax.py --ext=rb --cmd="ruby -c"
- mkdir build-travis - mkdir build-travis
- cd build-travis - cd build-travis
- cmake .. -G Ninja -DCMAKE_C_COMPILER=gcc-$GCC_VERSION -DCMAKE_CXX_COMPILER=g++-$GCC_VERSION -DDFHACK_BUILD_ARCH=64 -DBUILD_DOCS:BOOL=ON -DCMAKE_INSTALL_PREFIX="$DF_FOLDER" - cmake .. -G Ninja -DCMAKE_C_COMPILER=gcc-$GCC_VERSION -DCMAKE_CXX_COMPILER=g++-$GCC_VERSION -DDFHACK_BUILD_ARCH=64 -DBUILD_DOCS:BOOL=ON -DBUILD_TESTS:BOOL=ON -DCMAKE_INSTALL_PREFIX="$DF_FOLDER"
- ninja -j3 install - ninja -j3 install
- mv "$DF_FOLDER"/dfhack.init-example "$DF_FOLDER"/dfhack.init - mv "$DF_FOLDER"/dfhack.init-example "$DF_FOLDER"/dfhack.init
- cd .. - cd ..
- cp travis/dfhack_travis.init "$DF_FOLDER"/ - python travis/run-tests.py --headless --keep-status "$DF_FOLDER"
- python travis/run-tests.py "$DF_FOLDER"
- python travis/check-rpc.py "$DF_FOLDER/dfhack-rpc.txt" - python travis/check-rpc.py "$DF_FOLDER/dfhack-rpc.txt"
- cat "$DF_FOLDER/test_status.json"
before_cache: before_cache:
- cat "$DF_FOLDER/stderr.log" - cat "$DF_FOLDER/stderr.log"
- rm -rf "$DF_FOLDER" - rm -rf "$DF_FOLDER"

@ -1,9 +1,17 @@
local json = require 'json'
local script = require 'gui.script' local script = require 'gui.script'
local utils = require 'utils' local utils = require 'utils'
local args = {...} local args = {...}
local done_command = args[1] local done_command = args[1]
local STATUS_FILE = 'test_status.json'
local TestStatus = {
PENDING = 'pending',
PASSED = 'passed',
FAILED = 'failed',
}
expect = {} expect = {}
function expect.true_(value, comment) function expect.true_(value, comment)
return not not value, comment, 'expected true' return not not value, comment, 'expected true'
@ -78,18 +86,17 @@ function get_test_files()
return files return files
end end
function set_test_stage(stage) function load_test_status()
local f = io.open('test_stage.txt', 'w') if dfhack.filesystem.isfile(STATUS_FILE) then
f:write(stage) return json.decode_file(STATUS_FILE)
f:close() end
end end
function finish_tests(ok) function save_test_status(status)
if ok then json.encode_file(status, STATUS_FILE)
print('Tests finished') end
else
dfhack.printerr('Tests failed!') function finish_tests()
end
if done_command then if done_command then
dfhack.run_command(done_command) dfhack.run_command(done_command)
end end
@ -122,9 +129,11 @@ function main()
qerror('Could not find title screen') qerror('Could not find title screen')
end end
print('Running tests') print('Loading tests')
local tests = {}
for _, file in ipairs(files) do for _, file in ipairs(files) do
print('Running file: ' .. file:sub(file:find('test'), -1)) local short_filename = file:sub(file:find('test'), -1)
print('Loading file: ' .. short_filename)
local env, env_private = build_test_env() local env, env_private = build_test_env()
local code, err = loadfile(file, 't', env) local code, err = loadfile(file, 't', env)
if not code then if not code then
@ -138,35 +147,58 @@ function main()
counts.file_errors = counts.file_errors + 1 counts.file_errors = counts.file_errors + 1
dfhack.printerr('Error when running file: ' .. tostring(err)) dfhack.printerr('Error when running file: ' .. tostring(err))
else else
for name, test in pairs(env.test) do for name, test_func in pairs(env.test) do
env_private.checks = 0 local test_data = {
env_private.checks_ok = 0 full_name = short_filename .. ':' .. name,
counts.tests = counts.tests + 1 func = test_func,
local ok, err = pcall(test) private = env_private,
if not ok then }
dfhack.printerr('test errored: ' .. name .. ': ' .. tostring(err)) test_data.name = test_data.full_name:gsub('test/', ''):gsub('.lua', '')
passed = false table.insert(tests, test_data)
elseif env_private.checks ~= env_private.checks_ok then
dfhack.printerr('test failed: ' .. name)
passed = false
else
print('test passed: ' .. name)
counts.tests_ok = counts.tests_ok + 1
end
counts.checks = counts.checks + (tonumber(env_private.checks) or 0)
counts.checks_ok = counts.checks_ok + (tonumber(env_private.checks_ok) or 0)
end end
end end
end end
end end
local status = load_test_status() or {}
for _, test in pairs(tests) do
if not status[test.full_name] then
status[test.full_name] = TestStatus.PENDING
end
end
print('Running tests')
for _, test in pairs(tests) do
if status[test.full_name] == TestStatus.PENDING then
test.private.checks = 0
test.private.checks_ok = 0
counts.tests = counts.tests + 1
local ok, err = pcall(test.func)
local passed = false
if not ok then
dfhack.printerr('test errored: ' .. test.name .. ': ' .. tostring(err))
elseif test.private.checks ~= test.private.checks_ok then
dfhack.printerr('test failed: ' .. test.name)
else
print('test passed: ' .. test.name)
passed = true
counts.tests_ok = counts.tests_ok + 1
end
counts.checks = counts.checks + (tonumber(test.private.checks) or 0)
counts.checks_ok = counts.checks_ok + (tonumber(test.private.checks_ok) or 0)
status[test.full_name] = passed and TestStatus.PASSED or TestStatus.FAILED
save_test_status(status)
else
print('test skipped: ' .. test.name .. ' (state = ' .. status[test.full_name] .. ')')
end
end
print('\nTest summary:') print('\nTest summary:')
print(('%d/%d tests passed'):format(counts.tests_ok, counts.tests)) print(('%d/%d tests passed'):format(counts.tests_ok, counts.tests))
print(('%d/%d checks passed'):format(counts.checks_ok, counts.checks)) print(('%d/%d checks passed'):format(counts.checks_ok, counts.checks))
print(('%d test files failed to load'):format(counts.file_errors)) print(('%d test files failed to load'):format(counts.file_errors))
set_test_stage(passed and 'done' or 'fail') finish_tests()
finish_tests(passed)
end end
script.start(main) script.start(main)

@ -1,4 +0,0 @@
devel/dump-rpc dfhack-rpc.txt
:lua dfhack.internal.addScriptPath(dfhack.getHackPath())
test/main die

@ -36,10 +36,6 @@ mkdir df_linux
echo Extracting echo Extracting
tar xf "$tardest" --strip-components=1 -C df_linux tar xf "$tardest" --strip-components=1 -C df_linux
echo Changing settings
echo '' >> "df_linux/data/init/init.txt"
echo '[PRINT_MODE:TEXT]' >> "df_linux/data/init/init.txt"
echo '[SOUND:NO]' >> "df_linux/data/init/init.txt"
echo Done echo Done
echo "$selfmd5" > receipt echo "$selfmd5" > receipt

@ -1,40 +1,104 @@
import os, subprocess, sys import argparse
import enum
import json
import os
import re
import shutil
import subprocess
import sys
parser = argparse.ArgumentParser()
parser.add_argument('df_folder', help='DF base folder')
parser.add_argument('--headless', action='store_true',
help='Run without opening DF window (requires non-Windows)')
parser.add_argument('--keep-status', action='store_true',
help='Do not delete final status file')
args = parser.parse_args()
MAX_TRIES = 5 MAX_TRIES = 5
dfhack = 'Dwarf Fortress.exe' if sys.platform == 'win32' else './dfhack' dfhack = 'Dwarf Fortress.exe' if sys.platform == 'win32' else './dfhack'
test_stage = 'test_stage.txt' test_status_file = 'test_status.json'
class TestStatus(enum.Enum):
PENDING = 'pending'
PASSED = 'passed'
FAILED = 'failed'
def get_test_status():
if os.path.isfile(test_status_file):
with open(test_status_file) as f:
return {k: TestStatus(v) for k, v in json.load(f).items()}
def get_test_stage(): def change_setting(content, setting, value):
if os.path.isfile(test_stage): return '[' + setting + ':' + value + ']\n' + re.sub(
return open(test_stage).read().strip() r'\[' + setting + r':.+?\]', '(overridden)', content, flags=re.IGNORECASE)
return '0'
os.chdir(sys.argv[1]) os.chdir(sys.argv[1])
if os.path.exists(test_stage): if os.path.exists(test_status_file):
os.remove(test_stage) os.remove(test_status_file)
print(os.getcwd()) print('Backing up init.txt to init.txt.orig')
print(os.listdir('.')) init_txt_path = 'data/init/init.txt'
shutil.copyfile(init_txt_path, init_txt_path + '.orig')
tries = 0 with open(init_txt_path) as f:
while True: init_contents = f.read()
tries += 1 init_contents = change_setting(init_contents, 'INTRO', 'NO')
stage = get_test_stage() init_contents = change_setting(init_contents, 'SOUND', 'NO')
print('Run #%i: stage=%s' % (tries, get_test_stage())) init_contents = change_setting(init_contents, 'WINDOWED', 'YES')
if stage == 'done': init_contents = change_setting(init_contents, 'WINDOWEDX', '80')
print('Done!') init_contents = change_setting(init_contents, 'WINDOWEDY', '25')
os.remove(test_stage) init_contents = change_setting(init_contents, 'FPS', 'YES')
sys.exit(0) if args.headless:
elif stage == 'fail': init_contents = change_setting(init_contents, 'PRINT_MODE', 'TEXT')
print('Failed!')
os.remove(test_stage) test_init_file = 'dfhackzzz_test.init' # Core sorts these alphabetically
sys.exit(1) with open(test_init_file, 'w') as f:
if tries > MAX_TRIES: f.write('''
print('Too many tries - aborting') devel/dump-rpc dfhack-rpc.txt
sys.exit(1) :lua dfhack.internal.addScriptPath(dfhack.getHackPath())
test/main "lua scr.breakdown_level=df.interface_breakdown_types.QUIT"
process = subprocess.Popen([dfhack], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) ''')
process.communicate()
if process.returncode != 0: try:
print('DF exited with ' + repr(process.returncode)) with open(init_txt_path, 'w') as f:
f.write(init_contents)
tries = 0
while True:
status = get_test_status()
if status is not None:
if all(s is not TestStatus.PENDING for s in status.values()):
print('Done!')
break
elif tries > 0:
print('ERROR: Could not read status file')
sys.exit(2)
tries += 1
print('Starting DF: #%i' % (tries))
if tries > MAX_TRIES:
print('ERROR: Too many tries - aborting')
sys.exit(1)
if args.headless:
os.environ['DFHACK_HEADLESS'] = '1'
os.environ['DFHACK_DISABLE_CONSOLE'] = '1'
process = subprocess.Popen([dfhack],
stdin=subprocess.PIPE if args.headless else sys.stdin,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
_, err = process.communicate()
if err:
print('WARN: DF produced stderr: ' + repr(err[:5000]))
if process.returncode != 0:
print('WARN: DF exited with ' + repr(process.returncode))
finally:
print('\nRestoring original init.txt')
shutil.copyfile(init_txt_path + '.orig', init_txt_path)
if os.path.isfile(test_init_file):
os.remove(test_init_file)
if not args.keep_status and os.path.isfile(test_status_file):
os.remove(test_status_file)
print('Cleanup done')