Merge remote-tracking branch 'myk002/myk_printerr_match' into develop

develop
lethosor 2021-04-06 00:39:16 -04:00
commit 106fa4a5cf
No known key found for this signature in database
GPG Key ID: 76A269552F4F58C1
9 changed files with 420 additions and 216 deletions

@ -1,7 +1,7 @@
-- DFHack developer test harness
--@ module = true
local expect = require 'test.expect'
local expect = require 'test_util.expect'
local json = require 'json'
local script = require 'gui.script'
local utils = require 'utils'
@ -141,6 +141,33 @@ local function load_test_config(config_file)
return config
end
-- we have to save and use the original dfhack.printerr here so our test harness
-- output doesn't trigger its own dfhack.printerr usage detection (see
-- detect_printerr below)
local orig_printerr = dfhack.printerr
local function wrap_expect(func, private)
return function(...)
private.checks = private.checks + 1
local ret = {func(...)}
local ok = table.remove(ret, 1)
if ok then
private.checks_ok = private.checks_ok + 1
return
end
local msg = ''
for _, part in pairs(ret) do
if part then
msg = msg .. ': ' .. tostring(part)
end
end
msg = msg:sub(3) -- strip leading ': '
orig_printerr('Check failed! ' .. (msg or '(no message)'))
local info = debug.getinfo(2)
orig_printerr((' at %s:%d'):format(info.short_src, info.currentline))
print('')
end
end
local function build_test_env()
local env = {
test = utils.OrderedTable(),
@ -157,26 +184,7 @@ local function build_test_env()
checks_ok = 0,
}
for name, func in pairs(expect) do
env.expect[name] = function(...)
private.checks = private.checks + 1
local ret = {func(...)}
local ok = table.remove(ret, 1)
local msg = ''
for _, part in pairs(ret) do
if part then
msg = msg .. ': ' .. tostring(part)
end
end
msg = msg:sub(3) -- strip leading ': '
if ok then
private.checks_ok = private.checks_ok + 1
else
dfhack.printerr('Check failed! ' .. (msg or '(no message)'))
local info = debug.getinfo(2)
dfhack.printerr((' at %s:%d'):format(info.short_src, info.currentline))
print('')
end
end
env.expect[name] = wrap_expect(func, private)
end
setmetatable(env, {__index = _G})
return env, private
@ -258,12 +266,35 @@ local function sort_tests(tests)
end)
end
local function detect_printerr(func)
local saved_printerr = dfhack.printerr
local printerr_called = false
dfhack.printerr = function(msg)
if msg == nil then return end
saved_printerr(msg)
printerr_called = true
end
return dfhack.with_finalize(
function() dfhack.printerr = saved_printerr end,
function()
local ok, err = pcall(func)
if printerr_called then
return false,
"dfhack.printerr was called outside of" ..
" expect.printerr_match(). please wrap your test" ..
" with expect.printerr_match()."
end
return ok, err
end
)
end
local function run_test(test, status, counts)
test.private.checks = 0
test.private.checks_ok = 0
counts.tests = counts.tests + 1
dfhack.internal.IN_TEST = true
local ok, err = pcall(test.func)
local ok, err = detect_printerr(test.func)
dfhack.internal.IN_TEST = false
local passed = false
if not ok then

@ -47,6 +47,7 @@ changelog.txt uses a syntax similar to RST, with a few special sequences:
## Internals
- The DFHack test harness is now much easier to use for iterative development. Configuration can now be specified on the commandline, there are more test filter options, and the test harness can now easily rerun tests that have been run before.
- The ``test/main`` command to invoke the test harness has been renamed to just ``test``
- DFHack unit tests must now match any output expected to be printed via ``dfhack.printerr()``
# 0.47.05-r1

@ -1,145 +0,0 @@
-- Internal implementations of expect.*() functions for tests
-- NOTE: do not use this module in tests directly! The test wrapper (ci/test.lua)
-- wraps these functions to track which tests have passed/failed, and calling
-- these functions directly will not work as expected.
local expect = mkmodule('test.expect')
function expect.true_(value, comment)
return not not value, comment, 'expected true, got ' .. tostring(value)
end
function expect.false_(value, comment)
return not value, comment, 'expected false, got ' .. tostring(value)
end
function expect.fail(comment)
return false, comment or 'check failed, no reason provided'
end
function expect.nil_(value, comment)
return value == nil, comment, 'expected nil, got ' .. tostring(value)
end
function expect.eq(a, b, comment)
return a == b, comment, ('%s ~= %s'):format(a, b)
end
function expect.ne(a, b, comment)
return a ~= b, comment, ('%s == %s'):format(a, b)
end
function expect.lt(a, b, comment)
return a < b, comment, ('%s >= %s'):format(a, b)
end
function expect.le(a, b, comment)
return a <= b, comment, ('%s > %s'):format(a, b)
end
function expect.gt(a, b, comment)
return a > b, comment, ('%s <= %s'):format(a, b)
end
function expect.ge(a, b, comment)
return a >= b, comment, ('%s < %s'):format(a, b)
end
local function table_eq_recurse(a, b, keys, known_eq)
if a == b then return true end
local checked = {}
for k,v in pairs(a) do
if type(a[k]) == 'table' then
if known_eq[a[k]] and known_eq[a[k]][b[k]] then goto skip end
table.insert(keys, tostring(k))
if type(b[k]) ~= 'table' then
return false, keys, {tostring(a[k]), tostring(b[k])}
end
if not known_eq[a[k]] then known_eq[a[k]] = {} end
for eq_tab,_ in pairs(known_eq[a[k]]) do
known_eq[eq_tab][b[k]] = true
end
known_eq[a[k]][b[k]] = true
if not known_eq[b[k]] then known_eq[b[k]] = {} end
for eq_tab,_ in pairs(known_eq[b[k]]) do
known_eq[eq_tab][a[k]] = true
end
known_eq[b[k]][a[k]] = true
local matched, keys_at_diff, diff =
table_eq_recurse(a[k], b[k], keys, known_eq)
if not matched then return false, keys_at_diff, diff end
keys[#keys] = nil
elseif a[k] ~= b[k] then
table.insert(keys, tostring(k))
return false, keys, {tostring(a[k]), tostring(b[k])}
end
::skip::
checked[k] = true
end
for k in pairs(b) do
if not checked[k] then
table.insert(keys, tostring(k))
return false, keys, {tostring(a[k]), tostring(b[k])}
end
end
return true
end
function expect.table_eq(a, b, comment)
if (type(a) ~= 'table' and type(a) ~= 'userdata') or
(type(b) ~= 'table' and type(b) ~= 'userdata') then
return false, comment, 'operands to table_eq must be tables or userdata'
end
local keys, known_eq = {}, {}
local matched, keys_at_diff, diff = table_eq_recurse(a, b, keys, known_eq)
if matched then return true end
local keystr = '['..keys_at_diff[1]..']'
for i=2,#keys_at_diff do keystr = keystr..'['..keys_at_diff[i]..']' end
return false, comment,
('key %s: "%s" ~= "%s"'):format(keystr, diff[1], diff[2])
end
function expect.error(func, ...)
local ok, ret = pcall(func, ...)
if ok then
return false, 'no error raised by function call'
else
return true
end
end
function expect.error_match(func, matcher, ...)
local ok, err = pcall(func, ...)
if ok then
return false, 'no error raised by function call'
elseif type(matcher) == 'string' then
if not tostring(err):match(matcher) then
return false, ('error "%s" did not match "%s"'):format(err, matcher)
end
elseif not matcher(err) then
return false, ('error "%s" did not satisfy matcher'):format(err)
end
return true
end
function expect.pairs_contains(table, key, comment)
for k, v in pairs(table) do
if k == key then
return true
end
end
return false, comment, ('could not find key "%s" in table'):format(key)
end
function expect.not_pairs_contains(table, key, comment)
for k, v in pairs(table) do
if k == key then
return false, comment, ('found key "%s" in table'):format(key)
end
end
return true
end
return expect

@ -0,0 +1,234 @@
-- Internal implementations of expect.*() functions for tests
-- NOTE: do not use this module in tests directly! The test wrapper (ci/test.lua)
-- wraps these functions to track which tests have passed/failed, and calling
-- these functions directly will not work as expected.
local expect = mkmodule('test_util.expect')
function expect.true_(value, comment)
return not not value, comment, 'expected true, got ' .. tostring(value)
end
function expect.false_(value, comment)
return not value, comment, 'expected false, got ' .. tostring(value)
end
function expect.fail(comment)
return false, comment or 'check failed, no reason provided'
end
function expect.nil_(value, comment)
return value == nil, comment, 'expected nil, got ' .. tostring(value)
end
function expect.eq(a, b, comment)
return a == b, comment, ('%s ~= %s'):format(a, b)
end
function expect.ne(a, b, comment)
return a ~= b, comment, ('%s == %s'):format(a, b)
end
function expect.lt(a, b, comment)
return a < b, comment, ('%s >= %s'):format(a, b)
end
function expect.le(a, b, comment)
return a <= b, comment, ('%s > %s'):format(a, b)
end
function expect.gt(a, b, comment)
return a > b, comment, ('%s <= %s'):format(a, b)
end
function expect.ge(a, b, comment)
return a >= b, comment, ('%s < %s'):format(a, b)
end
local function table_eq_recurse(a, b, keys, known_eq)
if a == b then return true end
local checked = {}
for k,v in pairs(a) do
if type(a[k]) == 'table' then
if known_eq[a[k]] and known_eq[a[k]][b[k]] then goto skip end
table.insert(keys, tostring(k))
if type(b[k]) ~= 'table' then
return false, keys, {tostring(a[k]), tostring(b[k])}
end
if not known_eq[a[k]] then known_eq[a[k]] = {} end
for eq_tab,_ in pairs(known_eq[a[k]]) do
known_eq[eq_tab][b[k]] = true
end
known_eq[a[k]][b[k]] = true
if not known_eq[b[k]] then known_eq[b[k]] = {} end
for eq_tab,_ in pairs(known_eq[b[k]]) do
known_eq[eq_tab][a[k]] = true
end
known_eq[b[k]][a[k]] = true
local matched, keys_at_diff, diff =
table_eq_recurse(a[k], b[k], keys, known_eq)
if not matched then return false, keys_at_diff, diff end
keys[#keys] = nil
elseif a[k] ~= b[k] then
table.insert(keys, tostring(k))
return false, keys, {tostring(a[k]), tostring(b[k])}
end
::skip::
checked[k] = true
end
for k in pairs(b) do
if not checked[k] then
table.insert(keys, tostring(k))
return false, keys, {tostring(a[k]), tostring(b[k])}
end
end
return true
end
function expect.table_eq(a, b, comment)
if (type(a) ~= 'table' and type(a) ~= 'userdata') or
(type(b) ~= 'table' and type(b) ~= 'userdata') then
return false, comment, 'operands to table_eq must be tables or userdata'
end
local keys, known_eq = {}, {}
local matched, keys_at_diff, diff = table_eq_recurse(a, b, keys, known_eq)
if matched then return true end
local keystr = '['..keys_at_diff[1]..']'
for i=2,#keys_at_diff do keystr = keystr..'['..keys_at_diff[i]..']' end
return false, comment,
('key %s: "%s" ~= "%s"'):format(keystr, diff[1], diff[2])
end
local function matches(obj, matcher)
if type(matcher) == 'string' then
return tostring(obj):match(matcher)
elseif type(matcher) == 'function' then
return matcher(obj)
end
return false
end
-- matches errors thrown from the specified function. the check passes if an
-- error is thrown and the thrown error matches the specified matcher.
--
-- matcher can be:
-- a string, interpreted as a lua pattern that matches the error message
-- a function that takes an err object and returns a boolean (true means match)
function expect.error_match(matcher, func, comment)
local ok, err = pcall(func)
if ok then
return false, comment, 'no error raised by function call'
end
if matches(err, matcher) then return true end
local matcher_str = ''
if type(matcher) == 'string' then
matcher_str = (': "%s"'):format(matcher)
end
return false, comment,
('error "%s" did not satisfy matcher%s'):format(err, matcher_str)
end
function expect.error(func, comment)
return expect.error_match('.*', func, comment)
end
-- matches error messages output from dfhack.printerr() when the specified
-- callback is run. the check passes if all printerr messages are matched by
-- specified matchers and no matchers remain unmatched.
--
-- matcher can be:
-- a string, interpreted as a lua pattern that matches a message
-- a function that takes the string message and returns a boolean (true means
-- match)
-- a populated table that can be used to match multiple messages (explained
-- in more detail below)
--
-- if matcher is a table, it can contain:
-- a list of strings and/or functions which will be matched in order
-- a map of strings and/or functions to positive integers, which will be
-- matched (in any order) the number of times specified
--
-- when this function attempts to match a message, it will first try the next
-- matcher in the list (that is, the next numeric key). if that matcher doesn't
-- exist or doesn't match, it will try all string and function keys whose values
-- are numeric and > 0.
--
-- if a mapped matcher is matched, it will have its value decremented by 1.
function expect.printerr_match(matcher, func, comment)
local saved_printerr = dfhack.printerr
local messages = {}
dfhack.printerr = function(msg) table.insert(messages, msg) end
dfhack.with_finalize(
function() dfhack.printerr = saved_printerr end,
func)
if type(matcher) ~= 'table' then matcher = {matcher} end
while #messages > 0 do
local msg = messages[1]
if matches(msg, matcher[1]) then
table.remove(matcher, 1)
goto continue
end
for k,v in pairs(matcher) do
if type(v) == 'number' and v > 0 and matches(msg, k) then
local remaining = v - 1
if remaining == 0 then
matcher[k] = nil
else
matcher[k] = remaining
end
goto continue
end
end
break -- failed to match message
::continue::
table.remove(messages, 1)
end
local errmsgs, unmatched_messages, extra_matchers = {}, {}, {}
for _,msg in ipairs(messages) do
table.insert(unmatched_messages, ('"%s"'):format(msg))
end
if #unmatched_messages > 0 then
table.insert(errmsgs,
('unmatched dfhack.printerr() messages: %s'):format(
table.concat(unmatched_messages, ', ')))
end
for k,v in ipairs(matcher) do
table.insert(extra_matchers, ('"%s"'):format(tostring(v)))
matcher[k] = nil
end
for k,v in pairs(matcher) do
table.insert(extra_matchers,
('"%s"=%s'):format(tostring(k), tostring(v)))
end
if #extra_matchers > 0 then
table.insert(errmsgs,
('unmatched or invalid matchers: %s'):format(
table.concat(extra_matchers, ', ')))
end
if #errmsgs > 0 then
return false, comment, table.concat(errmsgs, '; ')
end
return true
end
function expect.pairs_contains(table, key, comment)
for k, v in pairs(table) do
if k == key then
return true
end
end
return false, comment, ('could not find key "%s" in table'):format(key)
end
function expect.not_pairs_contains(table, key, comment)
for k, v in pairs(table) do
if k == key then
return false, comment, ('found key "%s" in table'):format(key)
end
end
return true
end
return expect

@ -0,0 +1,116 @@
local expect_raw = require('test_util.expect')
function test.table_eq()
expect.true_(expect_raw.table_eq({}, {}))
expect.true_(expect_raw.table_eq({'a'}, {'a'}))
expect.true_(expect_raw.table_eq({{'a', k='val'}, 'b'},
{{'a', k='val'}, 'b'}))
expect.false_(expect_raw.table_eq(nil, nil)) -- operands must be non-nil
expect.false_(expect_raw.table_eq({}, nil))
expect.false_(expect_raw.table_eq(nil, {}))
expect.false_(expect_raw.table_eq({}, {''}))
expect.false_(expect_raw.table_eq({''}, {}))
expect.false_(expect_raw.table_eq({'a', {}}, {'a'}))
expect.false_(expect_raw.table_eq({{'a', k='val1'}, 'b'},
{{'a', k='val2'}, 'b'}))
local tab = {a='a', b='b'}
expect.true_(expect_raw.table_eq(tab, tab))
expect.true_(expect_raw.table_eq({tab}, {tab}))
local tab1, tab2 = {'a'}, {'a'}
tab1.self, tab2.self = tab1, tab2
expect.true_(expect_raw.table_eq(tab1, tab2))
tab1.other, tab2.other = tab2, tab1
expect.true_(expect_raw.table_eq(tab1, tab2))
local tabA, tabB, tabC, tabD = {k='a'}, {k='a'}, {k='a'}, {k='a'}
tabA.next, tabB.next, tabC.next, tabD.next = tabB, tabC, tabD, tabA
expect.true_(expect_raw.table_eq(tabA, tabB))
end
function test.error_match()
expect.true_(expect_raw.error_match('err', function() error('err0r') end))
expect.false_(expect_raw.error_match('err', function() end))
expect.false_(expect_raw.error_match('00', function() error('err0r') end))
expect.true_(expect_raw.error_match(
function() return true end, function() error('err0r') end))
expect.false_(expect_raw.error_match(
function() return false end, function() error('err0r') end))
end
function test.error()
expect.true_(expect_raw.error(function() error('err0r') end))
expect.true_(expect_raw.error(function() qerror('err0r') end))
expect.false_(expect_raw.error(function() end))
end
function test.printerr_match_printerr_restored()
local prev_printerr = dfhack.printerr
expect_raw.printerr_match('.*', function() dfhack.printerr('a') end)
expect.eq(prev_printerr, dfhack.printerr)
end
function test.printerr_match()
local noprint = function() end
local oneprint = function()
dfhack.printerr('a')
end
expect.true_(expect_raw.printerr_match(nil, noprint))
expect.true_(expect_raw.printerr_match({}, noprint))
expect.false_(expect_raw.printerr_match(nil, oneprint))
expect.false_(expect_raw.printerr_match({}, oneprint))
expect.true_(expect_raw.printerr_match('a', oneprint))
expect.true_(expect_raw.printerr_match({[1]='a'}, oneprint))
expect.true_(expect_raw.printerr_match({a=1}, oneprint))
expect.true_(expect_raw.printerr_match('.', oneprint))
expect.true_(expect_raw.printerr_match({['.']=1}, oneprint))
expect.true_(expect_raw.printerr_match('.*', oneprint))
expect.true_(expect_raw.printerr_match({['.*']=1}, oneprint))
expect.false_(expect_raw.printerr_match('b', oneprint))
expect.false_(expect_raw.printerr_match({b=1}, oneprint))
expect.false_(expect_raw.printerr_match({a=1,b=1}, oneprint))
local twoprint = function()
dfhack.printerr('a')
dfhack.printerr('b')
end
expect.true_(expect_raw.printerr_match({'a','b'}, twoprint))
expect.true_(expect_raw.printerr_match({a=1,b=1}, twoprint))
expect.false_(expect_raw.printerr_match({'b','a'}, twoprint))
expect.false_(expect_raw.printerr_match({a=1}, twoprint))
expect.false_(expect_raw.printerr_match({b=1}, twoprint))
expect.false_(expect_raw.printerr_match({a=1,b=2}, twoprint))
expect.false_(expect_raw.printerr_match({a=1,b=1,c=1}, twoprint))
local threeprint = function()
dfhack.printerr('a')
dfhack.printerr('b')
dfhack.printerr('c')
end
expect.true_(expect_raw.printerr_match({a=1,b=1,c=1}, threeprint))
local multiprint = function()
dfhack.printerr('a')
dfhack.printerr('b')
dfhack.printerr('a')
end
expect.true_(expect_raw.printerr_match({a=2,b=1}, multiprint))
expect.false_(expect_raw.printerr_match({a=1,b=1}, multiprint))
local nilprint = function()
dfhack.printerr()
end
expect.true_(expect_raw.printerr_match({}, nilprint))
local nilaprint = function()
dfhack.printerr()
dfhack.printerr('a')
end
expect.true_(expect_raw.printerr_match({'a'}, nilaprint))
end

@ -5,9 +5,9 @@ function test.index_name()
end
function test.index_name_bad()
expect.error_match(function()
expect.error_match('not found.$', function()
expect.eq(df.global.world.units.other.SOME_FAKE_NAME, 'container')
end, 'not found.$')
end)
end
function test.index_id()
@ -17,11 +17,11 @@ function test.index_id()
end
function test.index_id_bad()
expect.error_match(function()
expect.error_match('Cannot read field', function()
expect.eq(df.global.world.units.other[df.units_other_id._first_item - 1], 'container')
end, 'Cannot read field')
expect.error_match(function()
end)
expect.error_match('Cannot read field', function()
expect.eq(df.global.world.units.other[df.units_other_id._last_item + 1], 'container')
end, 'Cannot read field')
end)
end

@ -77,16 +77,16 @@ end
function test.index_read_negative()
with_temp_ref(function(x)
expect.error_match(function()
expect.error_match('negative index', function()
expect.true_(x:_displace(1)[-1])
end, 'negative index')
end)
end)
end
function test.index_write_negative()
with_temp_ref(function(x)
expect.error_match(function()
expect.error_match('negative index', function()
x:_displace(1)[-1] = 7
end, 'negative index')
end)
end)
end

@ -12,24 +12,24 @@ end
function test.get_non_primitive()
dfhack.with_temp_object(df.unit:new(), function(unit)
expect.error_match(function()
expect.error_match('not found', function()
return unit:_field('status').ref_target
end, 'not found')
end)
end)
end
function test.set()
dfhack.with_temp_object(df.unit:new(), function(unit)
expect.error_match(function()
expect.error_match('builtin property or method', function()
unit:_field('hist_figure_id').ref_target = df.coord
end, 'builtin property or method')
end)
end)
end
function test.set_non_primitive()
dfhack.with_temp_object(df.unit:new(), function(unit)
expect.error_match(function()
expect.error_match('not found', function()
unit:_field('status').ref_target = df.coord
end, 'not found')
end)
end)
end

@ -1,36 +1,3 @@
local expect_raw = require('test.expect')
function test.internal_in_test()
expect.true_(dfhack.internal.IN_TEST)
end
function test.table_eq()
expect.true_(expect_raw.table_eq({}, {}))
expect.true_(expect_raw.table_eq({'a'}, {'a'}))
expect.true_(expect_raw.table_eq({{'a', k='val'}, 'b'},
{{'a', k='val'}, 'b'}))
expect.false_(expect_raw.table_eq(nil, nil)) -- operands must be non-nil
expect.false_(expect_raw.table_eq({}, nil))
expect.false_(expect_raw.table_eq(nil, {}))
expect.false_(expect_raw.table_eq({}, {''}))
expect.false_(expect_raw.table_eq({''}, {}))
expect.false_(expect_raw.table_eq({'a', {}}, {'a'}))
expect.false_(expect_raw.table_eq({{'a', k='val1'}, 'b'},
{{'a', k='val2'}, 'b'}))
local tab = {a='a', b='b'}
expect.true_(expect_raw.table_eq(tab, tab))
expect.true_(expect_raw.table_eq({tab}, {tab}))
local tab1, tab2 = {'a'}, {'a'}
tab1.self, tab2.self = tab1, tab2
expect.true_(expect_raw.table_eq(tab1, tab2))
tab1.other, tab2.other = tab2, tab1
expect.true_(expect_raw.table_eq(tab1, tab2))
local tabA, tabB, tabC, tabD = {k='a'}, {k='a'}, {k='a'}, {k='a'}
tabA.next, tabB.next, tabC.next, tabD.next = tabB, tabC, tabD, tabA
expect.true_(expect_raw.table_eq(tabA, tabB))
end