2021-03-06 23:07:45 -07:00
|
|
|
-- DFHack developer test harness
|
|
|
|
--@ module = true
|
|
|
|
|
2023-08-03 21:16:48 -06:00
|
|
|
local expect = require('test_util.expect')
|
|
|
|
local helpdb = require('helpdb')
|
|
|
|
local json = require('json')
|
|
|
|
local mock = require('test_util.mock')
|
|
|
|
local script = require('gui.script')
|
|
|
|
local utils = require('utils')
|
2018-07-19 09:26:38 -06:00
|
|
|
|
2021-02-28 23:31:44 -07:00
|
|
|
local help_text =
|
2021-03-06 23:07:45 -07:00
|
|
|
[====[
|
|
|
|
|
2021-03-22 12:09:44 -06:00
|
|
|
test
|
|
|
|
====
|
2021-03-06 23:07:45 -07:00
|
|
|
|
2023-08-03 21:16:48 -06:00
|
|
|
Tags: dev
|
2021-02-28 23:31:44 -07:00
|
|
|
|
2023-08-03 21:16:48 -06:00
|
|
|
Command: "test"
|
|
|
|
|
|
|
|
Run DFHack regression tests.
|
|
|
|
|
|
|
|
Discover DFHack functionality that has broken due to recent changes in DF or DFHack.
|
|
|
|
|
|
|
|
Usage
|
|
|
|
-----
|
2021-03-07 03:51:29 -07:00
|
|
|
|
2021-03-22 12:09:44 -06:00
|
|
|
test [<options>] [<done_command>]
|
2021-03-07 03:51:29 -07:00
|
|
|
|
2021-03-07 09:51:02 -07:00
|
|
|
If a done_command is specified, it will be run after the tests complete.
|
2021-02-28 23:31:44 -07:00
|
|
|
|
2023-08-03 21:16:48 -06:00
|
|
|
Options
|
|
|
|
-------
|
|
|
|
|
|
|
|
-d, --test_dir specifies which directory to look in for tests. defaults to
|
|
|
|
the "hack/scripts/test" folder in your DF installation.
|
|
|
|
-m, --modes only run tests in the given comma separated list of modes.
|
|
|
|
see the next section for a list of valid modes. if not
|
|
|
|
specified, the tests are not filtered by modes.
|
|
|
|
-r, --resume skip tests that have already been run. remove the
|
|
|
|
test_status.json file to reset the record.
|
|
|
|
-s, --save_dir the save folder to load for "fortress" mode tests. this
|
|
|
|
save is only loaded if a fort is not already loaded when
|
|
|
|
a "fortress" mode test is run. if not specified, defaults to
|
|
|
|
'region1'.
|
|
|
|
-t, --tests only run tests that match one of the comma separated list of
|
|
|
|
patterns. if not specified, no tests are filtered and all tessts
|
|
|
|
are run.
|
|
|
|
|
|
|
|
Modes
|
|
|
|
-----
|
|
|
|
|
|
|
|
none the test can be run on any screen
|
|
|
|
title the test must be run on the DF title screen. note that if the game
|
|
|
|
has a map loaded, "title" mode tests cannot be run
|
|
|
|
fortress the test must be run while a map is loaded. if the game is
|
|
|
|
currently on the title screen, the save specified by the save_dir
|
|
|
|
parameter will be loaded.
|
|
|
|
|
|
|
|
Examples
|
|
|
|
--------
|
|
|
|
|
|
|
|
test runs all tests
|
|
|
|
test -r runs all tests that haven't been run before
|
|
|
|
test -m none runs tests that don't need the game to be in a
|
|
|
|
specific mode
|
|
|
|
test -t quickfort runs quickfort tests
|
|
|
|
test -d /path/to/dfhack-scripts/repo/test
|
|
|
|
runs tests in your dev scripts repo
|
2021-03-07 03:51:29 -07:00
|
|
|
|
|
|
|
Default values for the options may be set in a file named test_config.json in
|
|
|
|
your DF folder. Options with comma-separated values should be written as json
|
|
|
|
arrays. For example:
|
|
|
|
|
|
|
|
{
|
|
|
|
"test_dir": "/home/myk/src/dfhack-scripts/test",
|
2021-03-07 09:51:02 -07:00
|
|
|
"modes": [ "none" ],
|
|
|
|
"tests": [ "quickfort", "devel" ],
|
2021-03-22 11:06:03 -06:00
|
|
|
"done_command": "devel/luacov -c"
|
2021-03-07 03:51:29 -07:00
|
|
|
}
|
|
|
|
|
2021-03-06 23:07:45 -07:00
|
|
|
]====]
|
2018-07-18 12:29:13 -06:00
|
|
|
|
2020-03-31 22:26:51 -06:00
|
|
|
local CONFIG_FILE = 'test_config.json'
|
2020-03-24 23:43:09 -06:00
|
|
|
local STATUS_FILE = 'test_status.json'
|
2021-09-28 19:18:50 -06:00
|
|
|
local DF_STATE_FILE = 'test_df_state.json'
|
2020-03-24 23:43:09 -06:00
|
|
|
local TestStatus = {
|
|
|
|
PENDING = 'pending',
|
|
|
|
PASSED = 'passed',
|
|
|
|
FAILED = 'failed',
|
|
|
|
}
|
|
|
|
|
2021-03-06 23:07:45 -07:00
|
|
|
local function delay(frames)
|
2018-07-19 09:26:38 -06:00
|
|
|
frames = frames or 1
|
|
|
|
script.sleep(frames, 'frames')
|
|
|
|
end
|
|
|
|
|
2021-06-23 15:53:02 -06:00
|
|
|
-- Will call the predicate_fn every frame until it returns true. If it fails to
|
|
|
|
-- return true before timeout_frames have elapsed, throws an error. If
|
|
|
|
-- timeout_frames is not specified, defaults to 100.
|
|
|
|
local function delay_until(predicate_fn, timeout_frames)
|
|
|
|
timeout_frames = tonumber(timeout_frames) or 100
|
|
|
|
repeat
|
|
|
|
delay()
|
|
|
|
if predicate_fn() then return end
|
|
|
|
timeout_frames = timeout_frames - 1
|
|
|
|
until timeout_frames < 0
|
|
|
|
error('timed out while waiting for predicate to return true')
|
|
|
|
end
|
|
|
|
|
2021-03-06 23:07:45 -07:00
|
|
|
local function clean_require(module)
|
2020-12-15 19:27:04 -07:00
|
|
|
-- wrapper around require() - forces a clean load of every module to ensure
|
|
|
|
-- that modules checking for dfhack.internal.IN_TEST at load time behave
|
|
|
|
-- properly
|
|
|
|
if package.loaded[module] then
|
|
|
|
reload(module)
|
|
|
|
end
|
|
|
|
return require(module)
|
|
|
|
end
|
|
|
|
|
2021-04-06 12:33:49 -06:00
|
|
|
-- clean_run_script and clean_reqscript force a clean load of scripts directly
|
|
|
|
-- or indirectly included from the test file. we use our own scripts table
|
|
|
|
-- instead of the one in dfhack.internal so we don't affect the state scripts
|
|
|
|
-- that are used outside the test harness.
|
2021-03-29 13:28:19 -06:00
|
|
|
local test_scripts = {}
|
2021-03-29 12:26:28 -06:00
|
|
|
local test_envvars = {}
|
2021-04-06 12:33:49 -06:00
|
|
|
|
|
|
|
-- clean_run_script is accessed via the dfhack table, not directly from the env.
|
|
|
|
-- therefore we use this function in wrap_test() below and not in test_envvars.
|
|
|
|
local function clean_run_script(name, ...)
|
|
|
|
return dfhack.run_script_with_env(
|
|
|
|
test_envvars,
|
|
|
|
name,
|
|
|
|
{scripts=test_scripts},
|
|
|
|
...)
|
|
|
|
end
|
|
|
|
|
2021-03-22 14:29:47 -06:00
|
|
|
local function clean_reqscript(name)
|
2021-03-29 13:23:53 -06:00
|
|
|
local path = dfhack.findScript(name)
|
|
|
|
if test_scripts[path] then return test_scripts[path].env end
|
|
|
|
local _, env = dfhack.run_script_with_env(
|
|
|
|
test_envvars,
|
|
|
|
name,
|
|
|
|
{
|
|
|
|
scripts=test_scripts,
|
|
|
|
module=true,
|
|
|
|
module_strict=true
|
|
|
|
})
|
|
|
|
return env
|
2021-03-22 14:29:47 -06:00
|
|
|
end
|
2021-03-29 12:26:28 -06:00
|
|
|
test_envvars.require = clean_require
|
|
|
|
test_envvars.reqscript = clean_reqscript
|
2021-03-22 14:29:47 -06:00
|
|
|
|
2021-04-19 12:26:13 -06:00
|
|
|
local function is_title_screen(scr)
|
|
|
|
scr = scr or dfhack.gui.getCurViewscreen()
|
|
|
|
return df.viewscreen_titlest:is_instance(scr)
|
|
|
|
end
|
|
|
|
|
|
|
|
-- This only handles pre-fortress-load screens. It will time out if the player
|
|
|
|
-- has already loaded a fortress or is in any screen that can't get to the title
|
|
|
|
-- screen by sending ESC keys.
|
2021-03-06 23:07:45 -07:00
|
|
|
local function ensure_title_screen()
|
2021-04-19 12:26:13 -06:00
|
|
|
for i = 1, 100 do
|
2020-04-05 21:12:31 -06:00
|
|
|
local scr = dfhack.gui.getCurViewscreen()
|
2021-04-19 12:26:13 -06:00
|
|
|
if is_title_screen(scr) then
|
2020-04-05 21:12:31 -06:00
|
|
|
print('Found title screen')
|
2021-04-19 12:26:13 -06:00
|
|
|
return
|
|
|
|
end
|
|
|
|
scr:feed_key(df.interface_key.LEAVESCREEN)
|
|
|
|
delay(10)
|
|
|
|
if i % 10 == 0 then print('Looking for title screen...') end
|
|
|
|
end
|
|
|
|
qerror(string.format('Could not find title screen (timed out at %s)',
|
|
|
|
dfhack.gui.getCurFocus(true)))
|
|
|
|
end
|
|
|
|
|
|
|
|
local function is_fortress(focus_string)
|
|
|
|
focus_string = focus_string or dfhack.gui.getCurFocus(true)
|
|
|
|
return focus_string == 'dwarfmode/Default'
|
|
|
|
end
|
|
|
|
|
|
|
|
-- Requires that a fortress game is already loaded or is ready to be loaded via
|
|
|
|
-- the "Continue Playing" option in the title screen. Otherwise the function
|
|
|
|
-- will time out and/or exit with error.
|
2021-04-19 21:16:58 -06:00
|
|
|
local function ensure_fortress(config)
|
2021-04-19 12:26:13 -06:00
|
|
|
local focus_string = dfhack.gui.getCurFocus(true)
|
|
|
|
for screen_timeout = 1,10 do
|
|
|
|
if is_fortress(focus_string) then
|
|
|
|
print('Loaded fortress map')
|
|
|
|
-- pause the game (if it's not already paused)
|
|
|
|
dfhack.gui.resetDwarfmodeView(true)
|
|
|
|
return
|
|
|
|
end
|
2021-05-07 16:19:34 -06:00
|
|
|
local scr = dfhack.gui.getCurViewscreen(true)
|
2021-04-19 21:16:58 -06:00
|
|
|
if focus_string == 'title' or
|
|
|
|
focus_string == 'dfhack/lua/load_screen' then
|
|
|
|
-- qerror()'s on falure
|
|
|
|
dfhack.run_script('load-save', config.save_dir)
|
2021-04-19 12:26:13 -06:00
|
|
|
elseif focus_string ~= 'loadgame' then
|
2021-04-19 21:16:58 -06:00
|
|
|
-- if we're not actively loading a game, hope we're in
|
|
|
|
-- a screen where hitting ESC will get us to the game map
|
|
|
|
-- or the title screen
|
2020-04-05 21:12:31 -06:00
|
|
|
scr:feed_key(df.interface_key.LEAVESCREEN)
|
2021-04-19 12:26:13 -06:00
|
|
|
end
|
2021-04-19 21:16:58 -06:00
|
|
|
-- wait for current screen to change
|
2021-04-19 12:26:13 -06:00
|
|
|
local prev_focus_string = focus_string
|
|
|
|
for frame_timeout = 1,100 do
|
2020-04-05 21:12:31 -06:00
|
|
|
delay(10)
|
2021-04-19 12:26:13 -06:00
|
|
|
focus_string = dfhack.gui.getCurFocus(true)
|
|
|
|
if focus_string ~= prev_focus_string then
|
|
|
|
goto next_screen
|
|
|
|
end
|
|
|
|
if frame_timeout % 10 == 0 then
|
|
|
|
print(string.format(
|
|
|
|
'Loading fortress (currently at screen: %s)',
|
|
|
|
focus_string))
|
|
|
|
end
|
2020-04-05 21:12:31 -06:00
|
|
|
end
|
2021-04-19 12:26:13 -06:00
|
|
|
print('Timed out waiting for screen to change')
|
|
|
|
break
|
|
|
|
::next_screen::
|
2020-04-05 21:12:31 -06:00
|
|
|
end
|
2021-04-19 12:26:13 -06:00
|
|
|
qerror(string.format('Could not load fortress (timed out at %s)',
|
|
|
|
focus_string))
|
2020-04-05 21:12:31 -06:00
|
|
|
end
|
|
|
|
|
2021-04-19 12:26:13 -06:00
|
|
|
local MODES = {
|
|
|
|
none = {order=1, detect=function() return true end},
|
|
|
|
title = {order=2, detect=is_title_screen, navigate=ensure_title_screen},
|
|
|
|
fortress = {order=3, detect=is_fortress, navigate=ensure_fortress},
|
2020-04-05 21:12:31 -06:00
|
|
|
}
|
|
|
|
|
2021-03-06 23:07:45 -07:00
|
|
|
local function load_test_config(config_file)
|
2020-03-31 22:26:51 -06:00
|
|
|
local config = {}
|
|
|
|
if dfhack.filesystem.isfile(config_file) then
|
|
|
|
config = json.decode_file(config_file)
|
|
|
|
end
|
|
|
|
|
|
|
|
if not config.test_dir then
|
|
|
|
config.test_dir = dfhack.getHackPath() .. 'scripts/test'
|
|
|
|
end
|
|
|
|
|
2021-04-19 21:16:58 -06:00
|
|
|
if not config.save_dir then
|
|
|
|
config.save_dir = 'region1'
|
|
|
|
end
|
|
|
|
|
2020-03-31 22:26:51 -06:00
|
|
|
return config
|
|
|
|
end
|
|
|
|
|
2021-04-03 16:40:40 -06:00
|
|
|
-- we have to save and use the original dfhack.printerr here so our test harness
|
|
|
|
-- output doesn't trigger its own dfhack.printerr usage detection (see
|
|
|
|
-- detect_printerr below)
|
|
|
|
local orig_printerr = dfhack.printerr
|
2022-05-26 23:03:40 -06:00
|
|
|
local function wrap_expect(func, private, path)
|
2021-04-01 19:14:50 -06:00
|
|
|
return function(...)
|
|
|
|
private.checks = private.checks + 1
|
2021-04-03 16:40:40 -06:00
|
|
|
local ret = {func(...)}
|
2021-04-01 19:14:50 -06:00
|
|
|
local ok = table.remove(ret, 1)
|
|
|
|
if ok then
|
|
|
|
private.checks_ok = private.checks_ok + 1
|
|
|
|
return
|
|
|
|
end
|
|
|
|
local msg = ''
|
|
|
|
for _, part in pairs(ret) do
|
|
|
|
if part then
|
|
|
|
msg = msg .. ': ' .. tostring(part)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
msg = msg:sub(3) -- strip leading ': '
|
2021-04-03 16:40:40 -06:00
|
|
|
orig_printerr('Check failed! ' .. (msg or '(no message)'))
|
2021-04-17 11:55:43 -06:00
|
|
|
-- Generate a stack trace with all function calls in the same file as the caller to expect.*()
|
|
|
|
-- (this produces better stack traces when using helpers in tests)
|
|
|
|
local frame = 2
|
|
|
|
local caller_src
|
|
|
|
while true do
|
|
|
|
info = debug.getinfo(frame)
|
|
|
|
if not info then break end
|
|
|
|
if not caller_src then
|
|
|
|
caller_src = info.short_src
|
|
|
|
end
|
2021-06-23 22:39:03 -06:00
|
|
|
-- Skip any frames corresponding to C calls, or Lua functions defined in another file
|
|
|
|
-- these could include pcall(), with_finalize(), etc.
|
2022-05-26 23:03:40 -06:00
|
|
|
if info.what == 'Lua' and (info.short_src == caller_src or info.short_src == path) then
|
2021-04-17 11:55:43 -06:00
|
|
|
orig_printerr((' at %s:%d'):format(info.short_src, info.currentline))
|
|
|
|
end
|
|
|
|
frame = frame + 1
|
|
|
|
end
|
2021-04-01 19:14:50 -06:00
|
|
|
print('')
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2022-05-26 23:03:40 -06:00
|
|
|
local function build_test_env(path)
|
2018-07-18 12:29:13 -06:00
|
|
|
local env = {
|
2018-07-19 10:40:25 -06:00
|
|
|
test = utils.OrderedTable(),
|
2021-06-30 08:10:59 -06:00
|
|
|
-- config values can be overridden in the test file to define
|
|
|
|
-- requirements for the tests in that file
|
2020-03-26 22:34:44 -06:00
|
|
|
config = {
|
2021-06-30 08:10:59 -06:00
|
|
|
-- override with the required game mode
|
2020-03-26 22:34:44 -06:00
|
|
|
mode = 'none',
|
2021-06-30 08:10:59 -06:00
|
|
|
-- override with a test wrapper function with common setup and
|
|
|
|
-- teardown for every test, if needed. for example:
|
|
|
|
-- local function test_wrapper(test_fn)
|
|
|
|
-- mock.patch(dfhack.filesystem,
|
|
|
|
-- 'listdir_recursive',
|
|
|
|
-- mock.func({}),
|
|
|
|
-- test_fn)
|
|
|
|
-- end
|
|
|
|
-- config.wrapper = test_wrapper
|
|
|
|
wrapper = nil,
|
2020-03-26 22:34:44 -06:00
|
|
|
},
|
2018-07-18 12:29:13 -06:00
|
|
|
expect = {},
|
2021-04-08 22:26:33 -06:00
|
|
|
mock = mock,
|
2018-07-19 09:26:38 -06:00
|
|
|
delay = delay,
|
2021-06-23 15:53:02 -06:00
|
|
|
delay_until = delay_until,
|
2020-12-15 19:27:04 -07:00
|
|
|
require = clean_require,
|
2021-03-22 14:29:47 -06:00
|
|
|
reqscript = clean_reqscript,
|
2018-07-18 12:29:13 -06:00
|
|
|
}
|
|
|
|
local private = {
|
|
|
|
checks = 0,
|
|
|
|
checks_ok = 0,
|
|
|
|
}
|
|
|
|
for name, func in pairs(expect) do
|
2022-05-26 23:03:40 -06:00
|
|
|
env.expect[name] = wrap_expect(func, private, path)
|
2018-07-18 12:29:13 -06:00
|
|
|
end
|
|
|
|
setmetatable(env, {__index = _G})
|
|
|
|
return env, private
|
|
|
|
end
|
|
|
|
|
2021-03-06 23:07:45 -07:00
|
|
|
local function get_test_files(test_dir)
|
2018-07-18 12:29:13 -06:00
|
|
|
local files = {}
|
2021-03-06 23:07:45 -07:00
|
|
|
print('Loading tests from ' .. test_dir)
|
2020-03-31 22:26:51 -06:00
|
|
|
for _, entry in ipairs(dfhack.filesystem.listdir_recursive(test_dir)) do
|
2021-03-22 12:09:44 -06:00
|
|
|
if not entry.isdir then
|
2018-07-18 12:29:13 -06:00
|
|
|
table.insert(files, entry.path)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
table.sort(files)
|
|
|
|
return files
|
|
|
|
end
|
|
|
|
|
2021-03-06 23:07:45 -07:00
|
|
|
local function load_test_status()
|
|
|
|
if dfhack.filesystem.isfile(STATUS_FILE) then
|
2020-03-24 23:43:09 -06:00
|
|
|
return json.decode_file(STATUS_FILE)
|
|
|
|
end
|
2018-02-04 14:00:53 -07:00
|
|
|
end
|
|
|
|
|
2021-03-06 23:07:45 -07:00
|
|
|
local function save_test_status(status)
|
2020-03-24 23:43:09 -06:00
|
|
|
json.encode_file(status, STATUS_FILE)
|
|
|
|
end
|
|
|
|
|
2021-03-06 23:07:45 -07:00
|
|
|
local function finish_tests(done_command)
|
2020-12-15 19:24:38 -07:00
|
|
|
dfhack.internal.IN_TEST = false
|
2021-03-07 13:06:27 -07:00
|
|
|
if done_command and #done_command > 0 then
|
2018-07-18 12:29:13 -06:00
|
|
|
dfhack.run_command(done_command)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2021-03-06 23:07:45 -07:00
|
|
|
local function load_tests(file, tests)
|
2022-05-26 23:03:40 -06:00
|
|
|
local short_filename = file:sub((file:find('test') or -4) + 5, -1)
|
2020-03-26 21:26:43 -06:00
|
|
|
print('Loading file: ' .. short_filename)
|
2022-05-26 23:03:40 -06:00
|
|
|
local env, env_private = build_test_env(file)
|
2020-03-26 21:26:43 -06:00
|
|
|
local code, err = loadfile(file, 't', env)
|
|
|
|
if not code then
|
|
|
|
dfhack.printerr('Failed to load file: ' .. tostring(err))
|
|
|
|
return false
|
2023-08-03 21:16:48 -06:00
|
|
|
end
|
|
|
|
dfhack.internal.IN_TEST = true
|
|
|
|
local ok, err = dfhack.pcall(code)
|
|
|
|
dfhack.internal.IN_TEST = false
|
|
|
|
if not ok then
|
|
|
|
dfhack.printerr('Error when running file: ' .. tostring(err))
|
|
|
|
return false
|
|
|
|
end
|
|
|
|
if not MODES[env.config.mode] then
|
|
|
|
dfhack.printerr('Invalid config.mode: ' .. tostring(env.config.mode))
|
|
|
|
return false
|
|
|
|
end
|
2023-08-04 15:09:50 -06:00
|
|
|
if not env.config.target then
|
2023-08-03 21:48:15 -06:00
|
|
|
dfhack.printerr('Skipping tests for unspecified target in ' .. file)
|
|
|
|
return true -- TODO: change to false once existing tests have targets specified
|
2023-08-03 21:16:48 -06:00
|
|
|
end
|
2023-08-04 11:25:49 -06:00
|
|
|
local targets = type(env.config.target) == 'table' and env.config.target or {env.config.target}
|
2023-08-03 21:16:48 -06:00
|
|
|
for _,target in ipairs(targets) do
|
|
|
|
if target == 'core' then goto continue end
|
|
|
|
if type(target) ~= 'string' or not helpdb.is_entry(target) or
|
|
|
|
helpdb.get_entry_tags(target).unavailable
|
|
|
|
then
|
|
|
|
dfhack.printerr('Skipping tests for unavailable target: ' .. target)
|
2023-08-03 21:48:15 -06:00
|
|
|
return true
|
2020-03-26 21:26:43 -06:00
|
|
|
end
|
2023-08-03 21:16:48 -06:00
|
|
|
::continue::
|
|
|
|
end
|
|
|
|
for name, test_func in pairs(env.test) do
|
|
|
|
if env.config.wrapper then
|
|
|
|
local fn = test_func
|
|
|
|
test_func = function() env.config.wrapper(fn) end
|
|
|
|
end
|
|
|
|
local test_data = {
|
|
|
|
full_name = short_filename .. ':' .. name,
|
|
|
|
func = test_func,
|
|
|
|
private = env_private,
|
|
|
|
config = env.config,
|
|
|
|
}
|
|
|
|
test_data.name = test_data.full_name:gsub('test/', ''):gsub('.lua', '')
|
|
|
|
table.insert(tests, test_data)
|
2020-03-26 21:26:43 -06:00
|
|
|
end
|
|
|
|
return true
|
|
|
|
end
|
|
|
|
|
2021-03-06 23:07:45 -07:00
|
|
|
local function sort_tests(tests)
|
2020-04-05 21:12:31 -06:00
|
|
|
-- to make sort stable
|
|
|
|
local test_index = utils.invert(tests)
|
|
|
|
table.sort(tests, function(a, b)
|
|
|
|
if a.config.mode ~= b.config.mode then
|
2021-04-19 12:26:13 -06:00
|
|
|
return MODES[a.config.mode].order < MODES[b.config.mode].order
|
2020-04-05 21:12:31 -06:00
|
|
|
end
|
2021-04-19 12:26:13 -06:00
|
|
|
return test_index[a] < test_index[b]
|
2020-04-05 21:12:31 -06:00
|
|
|
end)
|
|
|
|
end
|
|
|
|
|
2021-04-06 12:33:49 -06:00
|
|
|
local function wrap_test(func)
|
2021-04-08 22:50:06 -06:00
|
|
|
local saved_printerr = dfhack.printerr
|
2021-04-03 16:40:40 -06:00
|
|
|
local printerr_called = false
|
2021-04-08 22:50:06 -06:00
|
|
|
local printerr_wrapper = function(msg)
|
|
|
|
if msg == nil then return end
|
|
|
|
saved_printerr(msg)
|
|
|
|
printerr_called = true
|
|
|
|
end
|
|
|
|
|
|
|
|
return mock.patch(
|
|
|
|
{
|
|
|
|
{dfhack, 'printerr', printerr_wrapper},
|
|
|
|
{dfhack, 'run_script', clean_run_script},
|
|
|
|
{dfhack, 'reqscript', clean_reqscript},
|
|
|
|
},
|
2021-04-03 16:40:40 -06:00
|
|
|
function()
|
2021-04-11 22:48:10 -06:00
|
|
|
local ok, err = dfhack.pcall(func)
|
2021-04-03 16:40:40 -06:00
|
|
|
if printerr_called then
|
|
|
|
return false,
|
|
|
|
"dfhack.printerr was called outside of" ..
|
|
|
|
" expect.printerr_match(). please wrap your test" ..
|
|
|
|
" with expect.printerr_match()."
|
|
|
|
end
|
|
|
|
return ok, err
|
|
|
|
end
|
|
|
|
)
|
|
|
|
end
|
|
|
|
|
2021-06-10 11:00:47 -06:00
|
|
|
local function get_elapsed_str(elapsed)
|
|
|
|
return elapsed < 1 and "<1" or tostring(elapsed)
|
|
|
|
end
|
|
|
|
|
2021-03-06 23:07:45 -07:00
|
|
|
local function run_test(test, status, counts)
|
2020-03-26 22:49:34 -06:00
|
|
|
test.private.checks = 0
|
|
|
|
test.private.checks_ok = 0
|
|
|
|
counts.tests = counts.tests + 1
|
2020-12-15 19:24:38 -07:00
|
|
|
dfhack.internal.IN_TEST = true
|
2021-06-10 10:50:54 -06:00
|
|
|
local start_ms = dfhack.getTickCount()
|
2021-04-06 12:33:49 -06:00
|
|
|
local ok, err = wrap_test(test.func)
|
2021-06-10 10:50:54 -06:00
|
|
|
local elapsed_ms = dfhack.getTickCount() - start_ms
|
2020-12-15 19:24:38 -07:00
|
|
|
dfhack.internal.IN_TEST = false
|
2020-03-26 22:49:34 -06:00
|
|
|
local passed = false
|
|
|
|
if not ok then
|
2021-04-11 22:48:10 -06:00
|
|
|
dfhack.printerr('error: ' .. tostring(err) .. '\n')
|
|
|
|
dfhack.printerr('test errored: ' .. test.name)
|
2020-03-26 22:49:34 -06:00
|
|
|
elseif test.private.checks ~= test.private.checks_ok then
|
|
|
|
dfhack.printerr('test failed: ' .. test.name)
|
2020-03-26 21:26:43 -06:00
|
|
|
else
|
2021-06-10 11:00:47 -06:00
|
|
|
local elapsed_str = get_elapsed_str(elapsed_ms)
|
|
|
|
print(('test passed in %s ms: %s'):format(elapsed_str, test.name))
|
2020-03-26 22:49:34 -06:00
|
|
|
passed = true
|
|
|
|
counts.tests_ok = counts.tests_ok + 1
|
2020-03-26 21:26:43 -06:00
|
|
|
end
|
2020-03-26 22:49:34 -06:00
|
|
|
counts.checks = counts.checks + (tonumber(test.private.checks) or 0)
|
|
|
|
counts.checks_ok = counts.checks_ok + (tonumber(test.private.checks_ok) or 0)
|
|
|
|
return passed
|
2020-03-26 21:26:43 -06:00
|
|
|
end
|
|
|
|
|
2021-03-06 23:07:45 -07:00
|
|
|
local function get_tests(test_files, counts)
|
2020-03-24 23:43:09 -06:00
|
|
|
local tests = {}
|
2021-03-06 23:07:45 -07:00
|
|
|
for _, file in ipairs(test_files) do
|
2020-03-26 21:26:43 -06:00
|
|
|
if not load_tests(file, tests) then
|
2018-07-18 12:29:13 -06:00
|
|
|
counts.file_errors = counts.file_errors + 1
|
|
|
|
end
|
|
|
|
end
|
2021-03-06 23:07:45 -07:00
|
|
|
return tests
|
|
|
|
end
|
2018-07-18 12:29:13 -06:00
|
|
|
|
2021-03-06 23:07:45 -07:00
|
|
|
local function filter_tests(tests, config)
|
2021-02-28 23:31:44 -07:00
|
|
|
if config.tests or config.modes then
|
2021-03-07 03:51:29 -07:00
|
|
|
for _,filter in ipairs({{'tests', 'name pattern'},{'modes', 'mode'}}) do
|
|
|
|
if config[filter[1]] and #config[filter[1]] > 0 then
|
|
|
|
print(string.format('Filtering tests by %s:', filter[2]))
|
|
|
|
for _,v in ipairs(config[filter[1]]) do
|
|
|
|
print(string.format(' %s', v))
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
2020-04-01 00:15:27 -06:00
|
|
|
local orig_length = #tests
|
|
|
|
for i = #tests, 1, -1 do
|
2021-02-28 23:31:44 -07:00
|
|
|
local remove = false
|
|
|
|
if config.modes then
|
|
|
|
remove = true
|
|
|
|
-- allow test if it matches any of the given modes
|
|
|
|
for _, mode in pairs(config.modes) do
|
|
|
|
if tests[i].config.mode == mode then
|
|
|
|
remove = false
|
|
|
|
break
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
if config.tests and not remove then
|
|
|
|
remove = true
|
|
|
|
-- allow test if it matches any of the given patterns
|
|
|
|
for _, pattern in pairs(config.tests) do
|
|
|
|
if tests[i].name:match(pattern) then
|
|
|
|
remove = false
|
|
|
|
break
|
|
|
|
end
|
2020-04-01 00:15:27 -06:00
|
|
|
end
|
|
|
|
end
|
2021-02-28 23:31:44 -07:00
|
|
|
if remove then table.remove(tests, i) end
|
2020-04-01 00:15:27 -06:00
|
|
|
end
|
|
|
|
print('Selected tests: ' .. #tests .. '/' .. orig_length)
|
|
|
|
end
|
2021-03-06 23:07:45 -07:00
|
|
|
|
|
|
|
local status = {}
|
2021-03-22 11:06:03 -06:00
|
|
|
if config.resume then
|
2021-03-06 23:07:45 -07:00
|
|
|
status = load_test_status() or status
|
|
|
|
for i = #tests, 1, -1 do
|
|
|
|
local test = tests[i]
|
|
|
|
if not status[test.full_name] then
|
|
|
|
status[test.full_name] = TestStatus.PENDING
|
|
|
|
elseif status[test.full_name] ~= TestStatus.PENDING then
|
2021-03-20 11:29:30 -06:00
|
|
|
print(('skipping test: %s: state = %s'):format(
|
2021-03-06 23:07:45 -07:00
|
|
|
test.name, status[test.full_name]))
|
|
|
|
table.remove(tests, i)
|
|
|
|
end
|
2020-03-24 23:43:09 -06:00
|
|
|
end
|
|
|
|
end
|
2021-03-06 23:07:45 -07:00
|
|
|
|
2020-04-05 21:12:31 -06:00
|
|
|
sort_tests(tests)
|
2021-03-06 23:07:45 -07:00
|
|
|
return status
|
|
|
|
end
|
2020-03-24 23:43:09 -06:00
|
|
|
|
2021-04-19 21:16:58 -06:00
|
|
|
local function run_tests(tests, status, counts, config)
|
2021-03-07 03:51:29 -07:00
|
|
|
print(('Running %d tests'):format(#tests))
|
2021-06-10 10:50:54 -06:00
|
|
|
local start_ms = dfhack.getTickCount()
|
2021-04-19 23:18:42 -06:00
|
|
|
local num_skipped = 0
|
2020-03-24 23:43:09 -06:00
|
|
|
for _, test in pairs(tests) do
|
2021-04-19 23:18:42 -06:00
|
|
|
if MODES[test.config.mode].failed then
|
|
|
|
num_skipped = num_skipped + 1
|
|
|
|
goto skip
|
|
|
|
end
|
2021-04-19 12:26:13 -06:00
|
|
|
if not MODES[test.config.mode].detect() then
|
2023-08-04 21:40:24 -06:00
|
|
|
print(('Switching to %s mode for test: %s'):format(test.config.mode, test.name))
|
2021-04-19 21:16:58 -06:00
|
|
|
local ok, err = pcall(MODES[test.config.mode].navigate, config)
|
2021-04-19 12:26:13 -06:00
|
|
|
if not ok then
|
|
|
|
MODES[test.config.mode].failed = true
|
|
|
|
dfhack.printerr(tostring(err))
|
2021-04-19 23:18:42 -06:00
|
|
|
num_skipped = num_skipped + 1
|
2021-04-19 12:26:13 -06:00
|
|
|
goto skip
|
|
|
|
end
|
|
|
|
end
|
2021-04-19 23:18:42 -06:00
|
|
|
if run_test(test, status, counts) then
|
|
|
|
status[test.full_name] = TestStatus.PASSED
|
|
|
|
else
|
|
|
|
status[test.full_name] = TestStatus.FAILED
|
|
|
|
end
|
2020-03-26 22:49:34 -06:00
|
|
|
save_test_status(status)
|
2021-04-19 23:18:42 -06:00
|
|
|
::skip::
|
2020-03-24 23:43:09 -06:00
|
|
|
end
|
2021-06-10 10:50:54 -06:00
|
|
|
local elapsed_ms = dfhack.getTickCount() - start_ms
|
2020-03-24 23:43:09 -06:00
|
|
|
|
2021-03-23 20:11:48 -06:00
|
|
|
local function print_summary_line(ok, message)
|
|
|
|
local print_fn = print
|
|
|
|
if not ok then
|
|
|
|
status['*'] = TestStatus.FAILED
|
|
|
|
print_fn = dfhack.printerr
|
|
|
|
end
|
2021-06-10 10:50:54 -06:00
|
|
|
print_fn(' ' .. message)
|
2021-03-23 20:11:48 -06:00
|
|
|
end
|
|
|
|
|
|
|
|
status['*'] = status['*'] or TestStatus.PASSED
|
2021-06-10 11:00:47 -06:00
|
|
|
print(('\nTests completed in %s ms:'):format(get_elapsed_str(elapsed_ms)))
|
2021-03-23 20:11:48 -06:00
|
|
|
print_summary_line(counts.tests_ok == counts.tests,
|
|
|
|
('%d/%d tests passed'):format(counts.tests_ok, counts.tests))
|
|
|
|
print_summary_line(counts.checks_ok == counts.checks,
|
|
|
|
('%d/%d checks passed'):format(counts.checks_ok, counts.checks))
|
|
|
|
print_summary_line(counts.file_errors == 0,
|
|
|
|
('%d test files failed to load'):format(counts.file_errors))
|
2021-04-19 23:18:42 -06:00
|
|
|
print_summary_line(num_skipped == 0,
|
|
|
|
('%d tests in unreachable modes'):format(num_skipped))
|
2021-03-23 20:11:48 -06:00
|
|
|
|
|
|
|
save_test_status(status)
|
2018-07-18 12:29:13 -06:00
|
|
|
end
|
|
|
|
|
2021-09-28 19:18:50 -06:00
|
|
|
local function dump_df_state()
|
|
|
|
local state = {
|
|
|
|
enabler = {
|
|
|
|
fps = df.global.enabler.fps,
|
|
|
|
gfps = df.global.enabler.gfps,
|
2023-08-03 21:16:48 -06:00
|
|
|
fullscreen_state = df.global.enabler.fullscreen_state.whole,
|
2021-09-28 19:18:50 -06:00
|
|
|
},
|
|
|
|
gps = {
|
|
|
|
dimx = df.global.gps.dimx,
|
|
|
|
dimy = df.global.gps.dimy,
|
|
|
|
display_frames = df.global.gps.display_frames,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
json.encode_file(state, DF_STATE_FILE)
|
|
|
|
print('DF global state: ' .. json.encode(state, {pretty = false}))
|
|
|
|
end
|
|
|
|
|
2021-03-06 23:07:45 -07:00
|
|
|
local function main(args)
|
2021-04-19 21:16:58 -06:00
|
|
|
local help, resume, test_dir, mode_filter, save_dir, test_filter =
|
|
|
|
false, false, nil, {}, nil, {}
|
2021-03-07 03:51:29 -07:00
|
|
|
local other_args = utils.processArgsGetopt(args, {
|
2021-03-06 23:07:45 -07:00
|
|
|
{'h', 'help', handler=function() help = true end},
|
|
|
|
{'d', 'test_dir', hasArg=true,
|
2021-04-19 21:16:58 -06:00
|
|
|
handler=function(arg) test_dir = arg end},
|
2021-03-06 23:07:45 -07:00
|
|
|
{'m', 'modes', hasArg=true,
|
2021-05-27 22:17:20 -06:00
|
|
|
handler=function(arg) mode_filter = utils.split_string(arg, ',') end},
|
2021-03-22 11:06:03 -06:00
|
|
|
{'r', 'resume', handler=function() resume = true end},
|
2021-04-19 21:16:58 -06:00
|
|
|
{'s', 'save_dir', hasArg=true,
|
|
|
|
handler=function(arg) save_dir = arg end},
|
2021-03-06 23:07:45 -07:00
|
|
|
{'t', 'tests', hasArg=true,
|
2021-05-27 22:17:20 -06:00
|
|
|
handler=function(arg) test_filter = utils.split_string(arg, ',') end},
|
2021-03-06 23:07:45 -07:00
|
|
|
})
|
|
|
|
|
|
|
|
if help then print(help_text) return end
|
|
|
|
|
2021-03-07 09:51:02 -07:00
|
|
|
local done_command = table.concat(other_args, ' ')
|
2021-03-06 23:07:45 -07:00
|
|
|
local config = load_test_config(CONFIG_FILE)
|
|
|
|
|
|
|
|
-- override config with any params specified on the commandline
|
|
|
|
if test_dir then config.test_dir = test_dir end
|
2021-03-22 11:06:03 -06:00
|
|
|
if resume then config.resume = true end
|
2021-04-19 21:16:58 -06:00
|
|
|
if save_dir then config.save_dir = save_dir end
|
2021-03-06 23:07:45 -07:00
|
|
|
if #mode_filter > 0 then config.modes = mode_filter end
|
|
|
|
if #test_filter > 0 then config.tests = test_filter end
|
2021-03-07 09:51:02 -07:00
|
|
|
if #done_command > 0 then config.done_command = done_command end
|
2021-03-06 23:07:45 -07:00
|
|
|
|
|
|
|
if not dfhack.filesystem.isdir(config.test_dir) then
|
|
|
|
qerror(('Invalid test folder: "%s"'):format(config.test_dir))
|
|
|
|
end
|
|
|
|
|
|
|
|
local counts = {
|
|
|
|
tests = 0,
|
|
|
|
tests_ok = 0,
|
|
|
|
checks = 0,
|
|
|
|
checks_ok = 0,
|
|
|
|
file_errors = 0,
|
|
|
|
}
|
|
|
|
|
2021-09-28 19:18:50 -06:00
|
|
|
dump_df_state()
|
2021-03-06 23:07:45 -07:00
|
|
|
local test_files = get_test_files(config.test_dir)
|
|
|
|
local tests = get_tests(test_files, counts)
|
|
|
|
local status = filter_tests(tests, config)
|
|
|
|
|
|
|
|
script.start(function()
|
|
|
|
dfhack.call_with_finalizer(1, true,
|
2021-03-07 09:51:02 -07:00
|
|
|
finish_tests, config.done_command,
|
2021-04-19 21:16:58 -06:00
|
|
|
run_tests, tests, status, counts, config)
|
2021-03-06 23:07:45 -07:00
|
|
|
end)
|
|
|
|
end
|
|
|
|
|
|
|
|
if not dfhack_flags.module then
|
|
|
|
main({...})
|
|
|
|
end
|