Attached is a patch that adds support for expected failures to
testing framework. IMHO currently testing framework has
several problem but this was probably the worst one. Namely,
without such support adding tests for known bugs was
inpractical: known failures were reported exactly the
same as new ones, so they would make checking for regressions
significantly harder.
The patch adds buch of xf* routines parallel to existing
ones -- the xf* routines expect failure, so they increment
separate counters for expected failures. I also added
a mulitline summary at the end, which hopefully will be
clearer than existing one.
--
Waldek Hebisch
[email protected]
Index: src/algebra/unittest.spad.pamphlet
===================================================================
--- src/algebra/unittest.spad.pamphlet (revision 758)
+++ src/algebra/unittest.spad.pamphlet (working copy)
@@ -94,6 +94,15 @@
++ incFail(s, o) is an internal function that steps the number of
++ failed tests and records in- and output.
+ incXfPass: (String, List OutputForm) -> Void
+ ++ incXfPass(s, o) is an internal function that steps the number
+ ++ of tests that passed but were expected to fail and records
+ ++ in- and output.
+
+ incXfFail:() -> Void
+ ++ incXfFail() is an internal function that steps the number
+ ++ expected failures.
+
chkLibraryError: () -> Void
++ chkLibraryError is an internal function that steps the number of
++ tests.
@@ -102,14 +111,22 @@
++ incLibraryError is an internal function that steps the number of
++ failed tests, but not the number of tests.
+ xfincLibraryError: (String, List OutputForm) -> Void
+ ++ like incLibraryError, but using expected failures.
+
incFatal: String -> Void
++ incFatal s is an internal function that steps the number of fatal
++ tests and records input.
+ xfincFatal: String -> Void
+ ++ like incFatal, but for expected failures.
+
decFatal: () -> Void
++ decFatal is an internal function that declares that the preceding
++ test did not fail with a fatal error.
+ xfdecFatal: () -> Void
+ ++ like decFatal, but for expected failures.
addWarning: OutputForm -> Void
++ addWarning s is an internal function that adds s, together with an
++ indication of the current testcase and the current testsuite to the
@@ -131,7 +148,10 @@
TESTCASE ==> Record(total: Integer,
fail: Integer,
+ xfail: Integer,
+ xfpassed: Integer,
fatalities: List FATAL,
+ xpassed_list: List FAILED,
failed: List FAILED)
TESTSUITE ==> AssociationList(String, TESTCASE)
@@ -165,7 +185,7 @@
currentTestcase := s
insert!([currentTestcase, _
- [0, 0, [], []]$TESTCASE]$Record(key: String, entry:
TESTCASE), _
+ [0, 0, 0, 0, [], [], []]$TESTCASE]$Record(key: String,
entry: TESTCASE), _
tests.currentTestsuite)
testcase s ==
@@ -189,6 +209,15 @@
-- number must not be stepped
cur.failed := cons([cur.total, inp, out]$FAILED, cur.failed)
+ xfincLibraryError(inp, out) ==
+ cur := tests.currentTestsuite.currentTestcase
+ cur.xfail := cur.xfail + 1
+
+ incXfFail() ==
+ cur := tests.currentTestsuite.currentTestcase
+ cur.xfail := cur.xfail + 1
+ cur.total := cur.total + 1
+
incFail(inp, out) ==
cur := tests.currentTestsuite.currentTestcase
cur.fail := cur.fail + 1
@@ -197,12 +226,24 @@
cur.failed := cons([cur.total, inp, out]$FAILED, cur.failed)
cur.total := cur.total + 1
+ incXfPass(inp, out) ==
+ cur := tests.currentTestsuite.currentTestcase
+ cur.xfpassed := cur.xfpassed + 1
+ cur.xpassed_list := cons([cur.total, inp, out]$FAILED,
+ cur.xpassed_list)
+ cur.total := cur.total + 1
+
incFatal inp ==
cur := tests.currentTestsuite.currentTestcase
cur.total := cur.total + 1
cur.fail := cur.fail + 1
cur.fatalities := cons([cur.total, inp]$FATAL, cur.fatalities)
+ xfincFatal inp ==
+ cur := tests.currentTestsuite.currentTestcase
+ cur.total := cur.total + 1
+ cur.xfail := cur.xfail + 1
+ cur.fatalities := cons([cur.total, inp]$FATAL, cur.fatalities)
decFatal() ==
cur := tests.currentTestsuite.currentTestcase
@@ -210,6 +251,11 @@
cur.fail := cur.fail - 1
cur.fatalities := rest(cur.fatalities)
+ xfdecFatal() ==
+ cur := tests.currentTestsuite.currentTestcase
+ cur.total := cur.total - 1
+ cur.xfail := cur.xfail - 1
+ cur.fatalities := rest(cur.fatalities)
addWarning s ==
idx := tests.currentTestsuite.currentTestcase.total
@@ -245,6 +291,12 @@
output(w)$P
output("")$P
+
+ totalTests: Integer := 0
+ failedTests: Integer := 0
+ xfailedTests: Integer := 0
+ xfpassedTests: Integer := 0
+
for tstsuite in reverse parts(tests)_
@List Record(key: String, entry: TESTSUITE) repeat
@@ -255,6 +307,11 @@
@List Record(key: String, entry: TESTCASE) repeat
totalCases := totalCases + 1
if tstcase.entry.fail > 0 then failedCases := failedCases + 1
+ totalTests := totalTests + tstcase.entry.total
+ failedTests := failedTests + tstcase.entry.fail
+ xfailedTests := xfailedTests + tstcase.entry.xfail
+ xfpassedTests := xfpassedTests + tstcase.entry.xfpassed
+
messagePrint(new(maxWidth, char "=")$String)$O
messagePrint("Testsuite: " tstsuite.key)$O
@@ -300,20 +357,20 @@
failedCases: Integer := 0
totalCases: Integer := 0
- failedTests: Integer := 0
- totalTests: Integer := 0
+ tsfailedTests: Integer := 0
+ tstotalTests: Integer := 0
for tstcase in reverse parts(tstsuite.entry)_
@List Record(key: String, entry: TESTCASE) repeat
totalCases := totalCases + 1
if tstcase.entry.fail > 0 then failedCases := failedCases + 1
- totalTests := totalTests + tstcase.entry.total
- failedTests := failedTests + tstcase.entry.fail
+ tstotalTests := tstotalTests + tstcase.entry.total
+ tsfailedTests := tsfailedTests + tstcase.entry.fail
sfailedCases := string failedCases
stotalCases := string totalCases
- sfailedTests := string failedTests
- stotalTests := string totalTests
+ sfailedTests := string tsfailedTests
+ stotalTests := string tstotalTests
messagePrint(tstsuite.key _
new(max(24-#(tstsuite.key),0)::NonNegativeInteger, _
char " ") _
@@ -325,8 +382,11 @@
char " ") sfailedTests _
new(max(6-#stotalTests, 0)::NonNegativeInteger, _
char " ") "(" stotalTests ")")$O
+ messagePrint("unexpected failures: " string failedTests)
+ messagePrint("expected failures: " string xfailedTests)
+ messagePrint("unexpected passes: " string xfpassedTests)
+ messagePrint("total tests: " string totalTests)
-
@
\section{package TESTAUX UnittestAux}
@@ -343,17 +403,32 @@
++ R performs some simplifications, we convert ex1 and ex2 to
++ \axiom{InputForm}, if possible.
+ xftestEqualsAux: (String, R, R) -> Void
+ ++ like testEquals, but expects failure.
+
testNotEqualsAux: (String, R, R) -> Void
++ testNotEquals(inp, ex1, ex2) states that ex1 and ex2 should be
++ different.
+ xftestNotEqualsAux: (String, R, R) -> Void
+ ++ like testEquals, but expects failure.
+
testTrueAux: (String, R) -> Void
++ testTrueAux(inp, ex) states that ex should be true.
+ xftestTrueAux: (String, R) -> Void
+ ++ like testEquals, but expects failure.
+
testLibraryErrorAux: (String, R) -> Void
++ testLibraryError(inp, ex) states that ex should throw an error. Such
++ a test will never count as a fatal error.
+ xftestLibraryErrorAux: (String, R) -> Void
+ ++ like testEquals, but expects failure.
+
+ testEqualsAuxCmp : (R, R) -> Boolean
+ ++ testEqualsAuxCmp should be local but is conditional
+
if R has RealNumberSystem then
testRealEqualsAux: (String, R, R) -> Void
++ testRealEquals(inp, ex1, ex2) states that ex1 and ex2 should be
@@ -361,12 +436,18 @@
++ \spadfun{testAbsolutePrecision} and
++ \spadfun{testRelativePrecision}.
+ xftestRealEqualsAux: (String, R, R) -> Void
+ ++ like testEquals, but expects failure.
+
testComplexEqualsAux: (String, Complex R, Complex R) -> Void
++ testComplexEquals(inp ex1, ex2) states that ex1 and ex2 should be
++ approximately equal as complex numbers, taking into acount
++ \spadfun{testAbsolutePrecision} and
++ \spadfun{testRelativePrecision}.
+ xftestComplexEqualsAux: (String, Complex R, Complex R) -> Void
+ ++ like testEquals, but expects failure.
+
testAbsolutePrecision: R -> R
++ testAbsolutePrecision(eps) returns the current absolute precision
++ used for floating point comparison, and then sets it to eps. The
@@ -382,24 +463,56 @@
O ==> OutputForm
U ==> UnittestCount
+ REPF ==> (String, List OutputForm) -> Void
+
+ nPasInc : REPF
+ nPasInc(s, o) == incPass()$U
+
+ nFailInc : REPF
+ nFailInc(s, o) == incFail(s, o)$U
+
+ xPasInc : REPF
+ xPasInc(s, o) == incXfPass(s, o)$U
+
+ xFailInc : REPF
+ xFailInc(s, o) == incXfFail()$U
+
+ testEqualsAux0 : (String, R, R, REPF, REPF) -> Void
+ -- testEqualsAuxCmp: (R, R) -> Boolean
+
if R has ConvertibleTo InputForm then
- testEqualsAux(inp, ex1, ex2) ==
- if (convert(ex1)@InputForm=convert(ex2)@InputForm)@Boolean
- then incPass()$U
- else
- if R has CoercibleTo O
- then incFail(inp, [ex1::O, ex2::O])$U
- else incFail(inp, [])$U
+
+ testEqualsAuxCmp(ex1, ex2) ==
+ (convert(ex1)@InputForm=convert(ex2)@InputForm)@Boolean
+
+ else if R is InputForm then
+
+ testEqualsAuxCmp(ex1, ex2) ==
+ (ex1=ex2)@Boolean
+
else
- testEqualsAux(inp, ex1, ex2) ==
+
+ testEqualsAuxCmp(ex1, ex2) ==
addWarning(message("testing in a domain without InputForm!")$O)$U
- if (ex1=ex2)@Boolean
- then incPass()$U
+ (ex1=ex2)@Boolean
+
+ testEqualsAux0(inp, ex1, ex2, prep, frep) ==
+ if testEqualsAuxCmp(ex1, ex2)
+ then
+ if R has CoercibleTo O
+ then prep(inp, [ex1::O, ex2::O])
+ else prep(inp, [])
else
if R has CoercibleTo O
- then incFail(inp, [ex1::O, ex2::O])$U
- else incFail(inp, [])$U
+ then frep(inp, [ex1::O, ex2::O])
+ else prep(inp, [])
+ testEqualsAux(inp, ex1, ex2) ==
+ testEqualsAux0(inp, ex1, ex2, nPasInc, nFailInc)
+
+ xftestEqualsAux(inp, ex1, ex2) ==
+ testEqualsAux0(inp, ex1, ex2, xPasInc, xFailInc)
+
if R has RealNumberSystem then
epsilonRelative := 10::R^(-15)
@@ -415,7 +528,9 @@
epsilonAbsolute := eps
old
- testRealEqualsAux(inp, ex1, ex2) ==
+ testRealEqualsAux0: (String, R, R, REPF, REPF) -> Void
+
+ testRealEqualsAux0(inp, ex1, ex2, prep, frep) ==
absolute: R := norm(ex1-ex2)$R
if not zero? ex2
then relative: R := norm(ex1/ex2-1)$R
@@ -434,10 +549,19 @@
else res := cons(message("abs err ")$O, res)
if empty? res
- then incPass()$U
- else incFail(inp, concat([ex1::O, ex2::O], res))$U
+ then prep(inp, [ex1::O, ex2::O])
+ else frep(inp, concat([ex1::O, ex2::O], res))
- testComplexEqualsAux(inp, ex1, ex2) ==
+ testRealEqualsAux(inp, ex1, ex2) ==
+ testRealEqualsAux0(inp, ex1, ex2, nPasInc, nFailInc)
+
+ xftestRealEqualsAux(inp, ex1, ex2) ==
+ testRealEqualsAux0(inp, ex1, ex2, xPasInc, xFailInc)
+
+ testComplexEqualsAux0: (String, Complex R, Complex R, REPF,
+ REPF) -> Void
+
+ testComplexEqualsAux0(inp, ex1, ex2, prep, frep) ==
absolute: R := norm(ex1-ex2)$Complex(R)
if not zero? ex2
then relative: R := norm(ex1/ex2-1)$Complex(R)
@@ -456,27 +580,40 @@
else res := cons(message("abs err ")$O, res)
if empty? res
- then incPass()$U
- else incFail(inp, concat([ex1::O, ex2::O], res))$U
+ then prep(inp, [ex1::O, ex2::O])
+ else frep(inp, concat([ex1::O, ex2::O], res))
+ testComplexEqualsAux(inp, ex1, ex2) ==
+ testComplexEqualsAux0(inp, ex1, ex2, nPasInc, nFailInc)
+
+ xftestComplexEqualsAux(inp, ex1, ex2) ==
+ testComplexEqualsAux0(inp, ex1, ex2, xPasInc, xFailInc)
+
+
testNotEqualsAux(inp, ex1, ex2) ==
- if (ex1=ex2)@Boolean
- then
- if R has CoercibleTo O
- then incFail(inp, [ex1::O, ex2::O])$U
- else incFail(inp, [])$U
- else incPass()$U
+ testEqualsAux0(inp, ex1, ex2, nFailInc, nPasInc)
- testTrueAux(inp, ex) ==
+ xftestNotEqualsAux(inp, ex1, ex2) ==
+ testEqualsAux0(inp, ex1, ex2, xFailInc, xPasInc)
+
+ testTrueAux0: (String, R, REPF, REPF) -> Void
+
+ testTrueAux0(inp, ex, prep, frep) ==
if R is Boolean then
if ex
- then incPass()$U
- else incFail(inp, [ex::Boolean::O])$U
+ then prep(inp, [ex::Boolean::O])
+ else frep(inp, [ex::Boolean::O])
else
if R has CoercibleTo O
- then incFail(inp, [ex::O])$U
- else incFail(inp, [])$U
+ then frep(inp, [ex::O])
+ else frep(inp, [])
+ testTrueAux(inp, ex) ==
+ testTrueAux0(inp, ex, nPasInc, nFailInc)
+
+ xftestTrueAux(inp, ex) ==
+ testTrueAux0(inp, ex, xPasInc, xFailInc)
+
testLibraryErrorAux(inp, ex) ==
-- at this point, ex was already evaluated. If there would have been an error,
-- incLibraryError would not be invoked.
@@ -484,6 +621,13 @@
then incLibraryError(inp, [ex::O])$U
else incLibraryError(inp, [])$U
+ xftestLibraryErrorAux(inp, ex) ==
+-- at this point, ex was already evaluated. If there would have been an error,
+-- incLibraryError would not be invoked.
+ if R has CoercibleTo O
+ then xfincLibraryError(inp, [ex::O])$U
+ else xfincLibraryError(inp, [])$U
+
@
\section{package TESTUNIT Unittest}
@@ -501,27 +645,45 @@
++ performs some simplifications, we convert ex1 and ex2 to
++ \axiom{InputForm}, if possible
+ xftestEquals: (String, String) -> Void
+ ++ like testEquals, but expects failure.
+
testRealEquals: (String, String) -> Void
++ testRealEquals(ex1, ex2) states that ex1 and ex2 should be
++ approximately equal as real numbers, taking into acount
++ \spadfun{testAbsolutePrecision} and \spadfun{testRelativePrecision}.
+ xftestRealEquals: (String, String) -> Void
+ ++ like testRealEquals, but expects failure.
+
testComplexEquals: (String, String) -> Void
++ testComplexEquals(ex1, ex2) states that ex1 and ex2 should be
++ approximately equal as complex numbers, taking into acount
++ \spadfun{testAbsolutePrecision} and \spadfun{testRelativePrecision}.
+ xftestComplexEquals: (String, String) -> Void
+ ++ like testComplexEquals, but expects failure.
+
testNotEquals: (String, String) -> Void
++ testNotEquals(ex1, ex2) states that ex1 and ex2 should be
++ different.
+ xftestNotEquals: (String, String) -> Void
+ ++ like testNotEquals, but expects failure.
+
testTrue: String -> Void
++ testTrue ex states that ex should be true.
+ xftestTrue: String -> Void
+ ++ like testTrue, but expects failure.
+
testLibraryError: String -> Void
++ testLibraryError ex states that ex should throw an error. Such a test
++ will never count as a fatal error.
+ xftestLibraryError: String -> Void
+ ++ like testLibraryError, but expects failure.
+
== add
T ==> TemplateUtilities
@@ -532,35 +694,69 @@
interpretString("testEqualsAux(_"" inp "_", " s1 ", " s2 ")")$T
decFatal()$UnittestCount
+ xftestEquals(s1, s2) ==
+ inp := "EQUL: (" s1 ", " s2 ")"
+ xfincFatal(inp)$UnittestCount
+ interpretString("xftestEqualsAux(_"" inp "_", " s1 ", " s2 ")")$T
+ xfdecFatal()$UnittestCount
+
testRealEquals(s1, s2) ==
inp := "EQLR: (" s1 ", " s2 ")"
incFatal(inp)$UnittestCount
interpretString("testRealEqualsAux(_"" inp "_", " s1 ", " s2 ")")$T
decFatal()$UnittestCount
+ xftestRealEquals(s1, s2) ==
+ inp := "EQLR: (" s1 ", " s2 ")"
+ xfincFatal(inp)$UnittestCount
+ interpretString("xftestRealEqualsAux(_"" inp "_", " s1 ", " s2 ")")$T
+ xfdecFatal()$UnittestCount
+
testComplexEquals(s1, s2) ==
inp := "EQLC: (" s1 ", " s2 ")"
incFatal(inp)$UnittestCount
interpretString("testComplexEqualsAux(_"" inp "_", " s1 ", " s2 ")")$T
decFatal()$UnittestCount
+ xftestComplexEquals(s1, s2) ==
+ inp := "EQLC: (" s1 ", " s2 ")"
+ xfincFatal(inp)$UnittestCount
+ interpretString("xftestComplexEqualsAux(_"" inp "_", " s1 ", " s2
")")$T
+ xfdecFatal()$UnittestCount
+
testNotEquals(s1, s2) ==
inp := "DIFF: (" s1 ", " s2 ")"
incFatal(inp)$UnittestCount
interpretString("testNotEqualsAux(_"" inp "_", " s1 ", " s2 ")")$T
decFatal()$UnittestCount
+ xftestNotEquals(s1, s2) ==
+ inp := "DIFF: (" s1 ", " s2 ")"
+ xfincFatal(inp)$UnittestCount
+ interpretString("xftestNotEqualsAux(_"" inp "_", " s1 ", " s2 ")")$T
+ xfdecFatal()$UnittestCount
+
testTrue(s) ==
inp := "TRUE: (" s ")"
incFatal(inp)$UnittestCount
interpretString("testTrueAux(_"" inp "_", " s ")")$T
decFatal()$UnittestCount
+ xftestTrue(s) ==
+ inp := "TRUE: (" s ")"
+ xfincFatal(inp)$UnittestCount
+ interpretString("xftestTrueAux(_"" inp "_", " s ")")$T
+ xfdecFatal()$UnittestCount
+
testLibraryError(s) ==
chkLibraryError()$UnittestCount
inp := "ERROR: (" s ")"
interpretString("testLibraryErrorAux(_"" inp "_", " s ")")$T
+ xftestLibraryError(s) ==
+ chkLibraryError()$UnittestCount
+ inp := "ERROR: (" s ")"
+ interpretString("xftestLibraryErrorAux(_"" inp "_", " s ")")$T
@
Index: src/input/bugs2009.input.pamphlet
===================================================================
--- src/input/bugs2009.input.pamphlet (revision 758)
+++ src/input/bugs2009.input.pamphlet (working copy)
@@ -34,7 +34,7 @@
testEquals("normalize(ii - asinh(2^x)/log(2))", "0")
testcase "asin complex(1.0,0) (issue #138)"
-testEquals("asin complex(1.0,0)", "pi()$Complex(Float)/2")
+xftestEquals("asin complex(1.0,0)", "pi()$Complex(Float)/2")
testcase "atanh 1.0 (issue #437)"
testLibraryError("atanh 1.0")
@@ -48,10 +48,22 @@
testTrue("not is?(operator first kernels res, 'integral)")
testEquals("normalize(D(res, " x ") - (" f "))", "0")
-testIntegrate("sqrt(1+tan(x)^2)", "x", "#436")
-testIntegrate("1/(-3+v^3)*v/(-4+v^3)^(1/2)", "v", "#360")
+xf1testIntegrate(f: String, x: String, issue: String): Void ==
+ testcaseNoClear("integrate(" f ", " x ") (" issue ")")
+ xftestTrue("(res := integrate(" f ", " x "); true)")
+ testTrue("not is?(operator first kernels res, 'integral)")
+ testEquals("normalize(D(res, " x ") - (" f "))", "0")
+
+xf2testIntegrate(f: String, x: String, issue: String): Void ==
+ testcaseNoClear("integrate(" f ", " x ") (" issue ")")
+ testTrue("(res := integrate(" f ", " x "); true)")
+ xftestTrue("not is?(operator first kernels res, 'integral)")
+ xftestEquals("normalize(D(res, " x ") - (" f "))", "0")
+
+xf1testIntegrate("sqrt(1+tan(x)^2)", "x", "#436")
+xf1testIntegrate("1/(-3+v^3)*v/(-4+v^3)^(1/2)", "v", "#360")
testIntegrate("log(1-z^3)*(%i*z)^(1/2)", "z", "#440")
-testIntegrate("csc(z)*(1-1/(%i*z)^(1/2))^(1/2)", "z", "#442")
+xf2testIntegrate("csc(z)*(1-1/(%i*z)^(1/2))^(1/2)", "z", "#442")
testIntegrate("asin(x/a)", "x", " ")
Index: src/input/bugs2007.input.pamphlet
===================================================================
--- src/input/bugs2007.input.pamphlet (revision 758)
+++ src/input/bugs2007.input.pamphlet (working copy)
@@ -55,7 +55,7 @@
"sqrt(2)/2*%pi")
strNoPole := "noPole"
-testEquals("eval(integrate(1/(a+x^4),x=0..%plusInfinity, strNoPole), a=1)", _
+xftestEquals("eval(integrate(1/(a+x^4),x=0..%plusInfinity, strNoPole), a=1)", _
"integrate(1/(1+x^4),x=0..%plusInfinity)")
testEquals("D(integrate((x^4+2*a*x^2+1)^-1, x=0..z, strNoPole), z)",
@@ -182,7 +182,7 @@
testTrue "abs(digamma(1/5::EXPR INT)::EXPR DFLOAT +
5.2890398965921882955472079624) < 10.0^(-13)"
testTrue "abs(besselJ(1,1::EXPR INT)::EXPR DFLOAT -
0.44005058574493351595968220371) < 10.0^(-13)"
-- fails, because airyAi yields complex results even for real arguments
-testTrue "abs(airyAi(1/5::EXPR INT)::EXPR DFLOAT -
0.303703154286381994892596723) < 10.0^(-13)"
+xftestTrue "abs(airyAi(1/5::EXPR INT)::EXPR DFLOAT -
0.303703154286381994892596723) < 10.0^(-13)"
testcase "issue 183"
bt l ==
--
You received this message because you are subscribed to the Google Groups
"FriCAS - computer algebra system" group.
To post to this group, send email to [email protected].
To unsubscribe from this group, send email to
[email protected].
For more options, visit this group at
http://groups.google.com/group/fricas-devel?hl=en.
