Overhaul lua testing script

- Now keeps track of the state of each test individually
- Only runs uncompleted tests if DF crashes/restarts
- DF now exits with 0
- Easier to run locally
- Hopefully works on Travis too!
develop
lethosor 2020-03-25 01:43:09 -04:00
parent 90d1f26504
commit ea3be02c63
5 changed files with 163 additions and 77 deletions

@ -32,8 +32,6 @@ before_install:
- pip install --user "sphinx==1.4" "requests[security]"
- sh travis/build-lua.sh
- sh travis/download-df.sh
- echo "export DFHACK_HEADLESS=1" >> "$HOME/.dfhackrc"
- echo "export DFHACK_DISABLE_CONSOLE=1" >> "$HOME/.dfhackrc"
script:
- export PATH="$PATH:$HOME/lua53/bin"
- git tag tmp-travis-build
@ -47,13 +45,13 @@ script:
- python travis/script-syntax.py --ext=rb --cmd="ruby -c"
- mkdir build-travis
- cd build-travis
- cmake .. -G Ninja -DCMAKE_C_COMPILER=gcc-$GCC_VERSION -DCMAKE_CXX_COMPILER=g++-$GCC_VERSION -DDFHACK_BUILD_ARCH=64 -DBUILD_DOCS:BOOL=ON -DCMAKE_INSTALL_PREFIX="$DF_FOLDER"
- cmake .. -G Ninja -DCMAKE_C_COMPILER=gcc-$GCC_VERSION -DCMAKE_CXX_COMPILER=g++-$GCC_VERSION -DDFHACK_BUILD_ARCH=64 -DBUILD_DOCS:BOOL=ON -DBUILD_TESTS:BOOL=ON -DCMAKE_INSTALL_PREFIX="$DF_FOLDER"
- ninja -j3 install
- mv "$DF_FOLDER"/dfhack.init-example "$DF_FOLDER"/dfhack.init
- cd ..
- cp travis/dfhack_travis.init "$DF_FOLDER"/
- python travis/run-tests.py "$DF_FOLDER"
- python travis/run-tests.py --headless --keep-status "$DF_FOLDER"
- python travis/check-rpc.py "$DF_FOLDER/dfhack-rpc.txt"
- cat "$DF_FOLDER/test_status.json"
before_cache:
- cat "$DF_FOLDER/stderr.log"
- rm -rf "$DF_FOLDER"

@ -1,9 +1,17 @@
local json = require 'json'
local script = require 'gui.script'
local utils = require 'utils'
local args = {...}
local done_command = args[1]
local STATUS_FILE = 'test_status.json'
local TestStatus = {
PENDING = 'pending',
PASSED = 'passed',
FAILED = 'failed',
}
expect = {}
function expect.true_(value, comment)
return not not value, comment, 'expected true'
@ -78,18 +86,17 @@ function get_test_files()
return files
end
function set_test_stage(stage)
local f = io.open('test_stage.txt', 'w')
f:write(stage)
f:close()
function load_test_status()
if dfhack.filesystem.isfile(STATUS_FILE) then
return json.decode_file(STATUS_FILE)
end
end
function finish_tests(ok)
if ok then
print('Tests finished')
else
dfhack.printerr('Tests failed!')
end
function save_test_status(status)
json.encode_file(status, STATUS_FILE)
end
function finish_tests()
if done_command then
dfhack.run_command(done_command)
end
@ -122,9 +129,11 @@ function main()
qerror('Could not find title screen')
end
print('Running tests')
print('Loading tests')
local tests = {}
for _, file in ipairs(files) do
print('Running file: ' .. file:sub(file:find('test'), -1))
local short_filename = file:sub(file:find('test'), -1)
print('Loading file: ' .. short_filename)
local env, env_private = build_test_env()
local code, err = loadfile(file, 't', env)
if not code then
@ -138,35 +147,58 @@ function main()
counts.file_errors = counts.file_errors + 1
dfhack.printerr('Error when running file: ' .. tostring(err))
else
for name, test in pairs(env.test) do
env_private.checks = 0
env_private.checks_ok = 0
counts.tests = counts.tests + 1
local ok, err = pcall(test)
if not ok then
dfhack.printerr('test errored: ' .. name .. ': ' .. tostring(err))
passed = false
elseif env_private.checks ~= env_private.checks_ok then
dfhack.printerr('test failed: ' .. name)
passed = false
else
print('test passed: ' .. name)
counts.tests_ok = counts.tests_ok + 1
end
counts.checks = counts.checks + (tonumber(env_private.checks) or 0)
counts.checks_ok = counts.checks_ok + (tonumber(env_private.checks_ok) or 0)
for name, test_func in pairs(env.test) do
local test_data = {
full_name = short_filename .. ':' .. name,
func = test_func,
private = env_private,
}
test_data.name = test_data.full_name:gsub('test/', ''):gsub('.lua', '')
table.insert(tests, test_data)
end
end
end
end
local status = load_test_status() or {}
for _, test in pairs(tests) do
if not status[test.full_name] then
status[test.full_name] = TestStatus.PENDING
end
end
print('Running tests')
for _, test in pairs(tests) do
if status[test.full_name] == TestStatus.PENDING then
test.private.checks = 0
test.private.checks_ok = 0
counts.tests = counts.tests + 1
local ok, err = pcall(test.func)
local passed = false
if not ok then
dfhack.printerr('test errored: ' .. test.name .. ': ' .. tostring(err))
elseif test.private.checks ~= test.private.checks_ok then
dfhack.printerr('test failed: ' .. test.name)
else
print('test passed: ' .. test.name)
passed = true
counts.tests_ok = counts.tests_ok + 1
end
counts.checks = counts.checks + (tonumber(test.private.checks) or 0)
counts.checks_ok = counts.checks_ok + (tonumber(test.private.checks_ok) or 0)
status[test.full_name] = passed and TestStatus.PASSED or TestStatus.FAILED
save_test_status(status)
else
print('test skipped: ' .. test.name .. ' (state = ' .. status[test.full_name] .. ')')
end
end
print('\nTest summary:')
print(('%d/%d tests passed'):format(counts.tests_ok, counts.tests))
print(('%d/%d checks passed'):format(counts.checks_ok, counts.checks))
print(('%d test files failed to load'):format(counts.file_errors))
set_test_stage(passed and 'done' or 'fail')
finish_tests(passed)
finish_tests()
end
script.start(main)

@ -1,4 +0,0 @@
devel/dump-rpc dfhack-rpc.txt
:lua dfhack.internal.addScriptPath(dfhack.getHackPath())
test/main die

@ -36,10 +36,6 @@ mkdir df_linux
echo Extracting
tar xf "$tardest" --strip-components=1 -C df_linux
echo Changing settings
echo '' >> "df_linux/data/init/init.txt"
echo '[PRINT_MODE:TEXT]' >> "df_linux/data/init/init.txt"
echo '[SOUND:NO]' >> "df_linux/data/init/init.txt"
echo Done
echo "$selfmd5" > receipt

@ -1,40 +1,104 @@
import os, subprocess, sys
import argparse
import enum
import json
import os
import re
import shutil
import subprocess
import sys
parser = argparse.ArgumentParser()
parser.add_argument('df_folder', help='DF base folder')
parser.add_argument('--headless', action='store_true',
help='Run without opening DF window (requires non-Windows)')
parser.add_argument('--keep-status', action='store_true',
help='Do not delete final status file')
args = parser.parse_args()
MAX_TRIES = 5
dfhack = 'Dwarf Fortress.exe' if sys.platform == 'win32' else './dfhack'
test_stage = 'test_stage.txt'
test_status_file = 'test_status.json'
class TestStatus(enum.Enum):
PENDING = 'pending'
PASSED = 'passed'
FAILED = 'failed'
def get_test_status():
if os.path.isfile(test_status_file):
with open(test_status_file) as f:
return {k: TestStatus(v) for k, v in json.load(f).items()}
def get_test_stage():
if os.path.isfile(test_stage):
return open(test_stage).read().strip()
return '0'
def change_setting(content, setting, value):
return '[' + setting + ':' + value + ']\n' + re.sub(
r'\[' + setting + r':.+?\]', '(overridden)', content, flags=re.IGNORECASE)
os.chdir(sys.argv[1])
if os.path.exists(test_stage):
os.remove(test_stage)
print(os.getcwd())
print(os.listdir('.'))
tries = 0
while True:
tries += 1
stage = get_test_stage()
print('Run #%i: stage=%s' % (tries, get_test_stage()))
if stage == 'done':
print('Done!')
os.remove(test_stage)
sys.exit(0)
elif stage == 'fail':
print('Failed!')
os.remove(test_stage)
sys.exit(1)
if tries > MAX_TRIES:
print('Too many tries - aborting')
sys.exit(1)
process = subprocess.Popen([dfhack], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
process.communicate()
if process.returncode != 0:
print('DF exited with ' + repr(process.returncode))
if os.path.exists(test_status_file):
os.remove(test_status_file)
print('Backing up init.txt to init.txt.orig')
init_txt_path = 'data/init/init.txt'
shutil.copyfile(init_txt_path, init_txt_path + '.orig')
with open(init_txt_path) as f:
init_contents = f.read()
init_contents = change_setting(init_contents, 'INTRO', 'NO')
init_contents = change_setting(init_contents, 'SOUND', 'NO')
init_contents = change_setting(init_contents, 'WINDOWED', 'YES')
init_contents = change_setting(init_contents, 'WINDOWEDX', '80')
init_contents = change_setting(init_contents, 'WINDOWEDY', '25')
init_contents = change_setting(init_contents, 'FPS', 'YES')
if args.headless:
init_contents = change_setting(init_contents, 'PRINT_MODE', 'TEXT')
test_init_file = 'dfhackzzz_test.init' # Core sorts these alphabetically
with open(test_init_file, 'w') as f:
f.write('''
devel/dump-rpc dfhack-rpc.txt
:lua dfhack.internal.addScriptPath(dfhack.getHackPath())
test/main "lua scr.breakdown_level=df.interface_breakdown_types.QUIT"
''')
try:
with open(init_txt_path, 'w') as f:
f.write(init_contents)
tries = 0
while True:
status = get_test_status()
if status is not None:
if all(s is not TestStatus.PENDING for s in status.values()):
print('Done!')
break
elif tries > 0:
print('ERROR: Could not read status file')
sys.exit(2)
tries += 1
print('Starting DF: #%i' % (tries))
if tries > MAX_TRIES:
print('ERROR: Too many tries - aborting')
sys.exit(1)
if args.headless:
os.environ['DFHACK_HEADLESS'] = '1'
os.environ['DFHACK_DISABLE_CONSOLE'] = '1'
process = subprocess.Popen([dfhack],
stdin=subprocess.PIPE if args.headless else sys.stdin,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
_, err = process.communicate()
if err:
print('WARN: DF produced stderr: ' + repr(err[:5000]))
if process.returncode != 0:
print('WARN: DF exited with ' + repr(process.returncode))
finally:
print('\nRestoring original init.txt')
shutil.copyfile(init_txt_path + '.orig', init_txt_path)
if os.path.isfile(test_init_file):
os.remove(test_init_file)
if not args.keep_status and os.path.isfile(test_status_file):
os.remove(test_status_file)
print('Cleanup done')