This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "CMake".
The branch, master has been updated
via 6660926f22b940759954e74e2b4c0c3dd0ff1991 (commit)
via 39ac8b4eb5c5ea99cf1053bd37e76d811f5122fc (commit)
via 80c2c9d14cf1c1a8f162e119bd00d5f483a94af2 (commit)
via 0187e522448c8fe92c15d4a983afac36950d1d59 (commit)
from b42da462333aea986576d00bf9af7449aa6e8010 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
https://cmake.org/gitweb?p=cmake.git;a=commitdiff;h=6660926f22b940759954e74e2b4c0c3dd0ff1991
commit 6660926f22b940759954e74e2b4c0c3dd0ff1991
Merge: b42da46233 39ac8b4eb5
Author: Brad King <brad.k...@kitware.com>
AuthorDate: Fri Nov 1 13:55:09 2019 +0000
Commit: Kitware Robot <kwro...@kitware.com>
CommitDate: Fri Nov 1 09:55:18 2019 -0400
Merge topic 'ctest-repeat-until-pass'
39ac8b4eb5 ctest: Add --repeat-after-timeout option
80c2c9d14c ctest: Add --repeat-until-pass option
0187e52244 cmCTestRunTest: Use inline member initializers
Acked-by: Kitware Robot <kwro...@kitware.com>
Merge-request: !3960
https://cmake.org/gitweb?p=cmake.git;a=commitdiff;h=39ac8b4eb5c5ea99cf1053bd37e76d811f5122fc
commit 39ac8b4eb5c5ea99cf1053bd37e76d811f5122fc
Author: Brad King <brad.k...@kitware.com>
AuthorDate: Tue Oct 29 14:21:38 2019 -0400
Commit: Brad King <brad.k...@kitware.com>
CommitDate: Tue Oct 29 15:14:36 2019 -0400
ctest: Add --repeat-after-timeout option
Add an option to re-run tests if they timeout. This will help tolerate
sporadic timeouts on busy machines.
diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst
index 031b9a7636..0097cee148 100644
--- a/Help/manual/ctest.1.rst
+++ b/Help/manual/ctest.1.rst
@@ -268,9 +268,17 @@ Options
``--repeat-until-pass <n>``
Allow each test to run up to ``<n>`` times in order to pass.
+ Repeats tests if they fail for any reason.
This is useful in tolerating sporadic failures in test cases.
+``--repeat-after-timeout <n>``
+ Allow each test to run up to ``<n>`` times in order to pass.
+ Repeats tests only if they timeout.
+
+ This is useful in tolerating sporadic timeouts in test cases
+ on busy machines.
+
``--max-width <width>``
Set the max width for a test name to output.
diff --git a/Help/release/dev/ctest-repeat-until-pass.rst
b/Help/release/dev/ctest-repeat-until-pass.rst
index 003cdf2aec..d177247181 100644
--- a/Help/release/dev/ctest-repeat-until-pass.rst
+++ b/Help/release/dev/ctest-repeat-until-pass.rst
@@ -1,5 +1,6 @@
ctest-repeat-until-pass
-----------------------
-* The :manual:`ctest(1)` tool learned a new ``--repeat-until-pass <n>``
- option to help tolerate sporadic test failures.
+* The :manual:`ctest(1)` tool learned new ``--repeat-until-pass <n>``
+ and ``--repeat-after-timeout <n>`` options to help tolerate sporadic
+ test failures.
diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx
index ce9e13bbba..ba146532b6 100644
--- a/Source/CTest/cmCTestRunTest.cxx
+++ b/Source/CTest/cmCTestRunTest.cxx
@@ -345,7 +345,9 @@ bool cmCTestRunTest::NeedsToRerun()
if ((this->RerunMode == cmCTest::Rerun::UntilFail &&
this->TestResult.Status == cmCTestTestHandler::COMPLETED) ||
(this->RerunMode == cmCTest::Rerun::UntilPass &&
- this->TestResult.Status != cmCTestTestHandler::COMPLETED)) {
+ this->TestResult.Status != cmCTestTestHandler::COMPLETED) ||
+ (this->RerunMode == cmCTest::Rerun::AfterTimeout &&
+ this->TestResult.Status == cmCTestTestHandler::TIMEOUT)) {
this->RunAgain = true;
return true;
}
@@ -745,10 +747,11 @@ void cmCTestRunTest::WriteLogOutputTop(size_t completed,
size_t total)
// then it will never print out the completed / total, same would
// got for run until pass. Trick is when this is called we don't
// yet know if we are passing or failing.
- if ((this->RerunMode != cmCTest::Rerun::UntilPass &&
- this->NumberOfRunsLeft == 1) ||
- (this->RerunMode == cmCTest::Rerun::UntilPass &&
- this->NumberOfRunsLeft == this->NumberOfRunsTotal) ||
+ bool const progressOnLast =
+ (this->RerunMode != cmCTest::Rerun::UntilPass &&
+ this->RerunMode != cmCTest::Rerun::AfterTimeout);
+ if ((progressOnLast && this->NumberOfRunsLeft == 1) ||
+ (!progressOnLast && this->NumberOfRunsLeft == this->NumberOfRunsTotal) ||
this->CTest->GetTestProgressOutput()) {
outputStream << std::setw(getNumWidth(total)) << completed << "/";
outputStream << std::setw(getNumWidth(total)) << total << " ";
diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx
index 7276d98fba..20445b0853 100644
--- a/Source/cmCTest.cxx
+++ b/Source/cmCTest.cxx
@@ -1884,6 +1884,28 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
}
}
+ if (this->CheckArgument(arg, "--repeat-after-timeout")) {
+ if (i >= args.size() - 1) {
+ errormsg = "'--repeat-after-timeout' requires an argument";
+ return false;
+ }
+ if (this->Impl->RerunMode != cmCTest::Rerun::Never) {
+ errormsg = "At most one '--repeat-*' option may be used.";
+ return false;
+ }
+ i++;
+ long repeat = 1;
+ if (!cmStrToLong(args[i], &repeat)) {
+ errormsg =
+ "'--repeat-after-timeout' given non-integer value '" + args[i] + "'";
+ return false;
+ }
+ this->Impl->RepeatTests = static_cast<int>(repeat);
+ if (repeat > 1) {
+ this->Impl->RerunMode = cmCTest::Rerun::AfterTimeout;
+ }
+ }
+
if (this->CheckArgument(arg, "--test-load") && i < args.size() - 1) {
i++;
unsigned long load;
diff --git a/Source/cmCTest.h b/Source/cmCTest.h
index c6b89281d7..bef0f8d368 100644
--- a/Source/cmCTest.h
+++ b/Source/cmCTest.h
@@ -438,6 +438,7 @@ public:
Never,
UntilFail,
UntilPass,
+ AfterTimeout,
};
Rerun GetRerunMode() const;
diff --git a/Source/ctest.cxx b/Source/ctest.cxx
index f716d7a941..2659e3071b 100644
--- a/Source/ctest.cxx
+++ b/Source/ctest.cxx
@@ -102,6 +102,8 @@ static const char* cmDocumentationOptions[][2] = {
"Require each test to run <n> times without failing in order to pass" },
{ "--repeat-until-pass <n>",
"Allow each test to run up to <n> times in order to pass" },
+ { "--repeat-after-timeout <n>",
+ "Allow each test to run up to <n> times if it times out" },
{ "--max-width <width>", "Set the max width for a test name to output" },
{ "--interactive-debug-mode [0|1]", "Set the interactive mode to 0 or 1." },
{ "--hardware-spec-file <file>", "Set the hardware spec file to use." },
diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
index 15a90fc02f..d7f41336e8 100644
--- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
+++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
@@ -24,12 +24,25 @@ run_cmake_command(repeat-until-fail-good
${CMAKE_CTEST_COMMAND} --repeat-until-fail 2
)
+run_cmake_command(repeat-after-timeout-bad1
+ ${CMAKE_CTEST_COMMAND} --repeat-after-timeout
+ )
+run_cmake_command(repeat-after-timeout-bad2
+ ${CMAKE_CTEST_COMMAND} --repeat-after-timeout foo
+ )
+run_cmake_command(repeat-after-timeout-good
+ ${CMAKE_CTEST_COMMAND} --repeat-after-timeout 2
+ )
+
run_cmake_command(repeat-until-pass-and-fail
${CMAKE_CTEST_COMMAND} --repeat-until-pass 2 --repeat-until-fail 2
)
run_cmake_command(repeat-until-fail-and-pass
${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat-until-pass 2
)
+run_cmake_command(repeat-until-fail-and-timeout
+ ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat-after-timeout 2
+ )
function(run_repeat_until_pass_tests)
# Use a single build tree for a few tests without cleaning.
@@ -42,6 +55,17 @@ function(run_repeat_until_pass_tests)
endfunction()
run_repeat_until_pass_tests()
+function(run_repeat_after_timeout_tests)
+ # Use a single build tree for a few tests without cleaning.
+ set(RunCMake_TEST_BINARY_DIR
${RunCMake_BINARY_DIR}/repeat-after-timeout-build)
+ run_cmake(repeat-after-timeout-cmake)
+ set(RunCMake_TEST_NO_CLEAN 1)
+ run_cmake_command(repeat-after-timeout-ctest
+ ${CMAKE_CTEST_COMMAND} -C Debug --repeat-after-timeout 3
+ )
+endfunction()
+run_repeat_after_timeout_tests()
+
function(run_repeat_until_fail_tests)
# Use a single build tree for a few tests without cleaning.
set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-fail-build)
diff --git
a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-result.txt
b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-result.txt
new file mode 100644
index 0000000000..d00491fd7e
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-result.txt
@@ -0,0 +1 @@
+1
diff --git
a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-stderr.txt
b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-stderr.txt
new file mode 100644
index 0000000000..aea92b8bb0
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-stderr.txt
@@ -0,0 +1 @@
+^CMake Error: '--repeat-after-timeout' requires an argument$
diff --git
a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-result.txt
b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-result.txt
new file mode 100644
index 0000000000..d00491fd7e
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-result.txt
@@ -0,0 +1 @@
+1
diff --git
a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-stderr.txt
b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-stderr.txt
new file mode 100644
index 0000000000..c5db55b65b
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-stderr.txt
@@ -0,0 +1 @@
+^CMake Error: '--repeat-after-timeout' given non-integer value 'foo'$
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-cmake.cmake
b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-cmake.cmake
new file mode 100644
index 0000000000..4dc9c3648f
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-cmake.cmake
@@ -0,0 +1,15 @@
+enable_testing()
+
+set(TEST_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/test_output.txt")
+add_test(NAME initialization
+ COMMAND ${CMAKE_COMMAND}
+ "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
+ -P "${CMAKE_CURRENT_SOURCE_DIR}/init.cmake")
+add_test(NAME test1
+ COMMAND ${CMAKE_COMMAND}
+ "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
+ -P "${CMAKE_CURRENT_SOURCE_DIR}/test1-timeout.cmake")
+set_tests_properties(test1 PROPERTIES DEPENDS "initialization" TIMEOUT 2)
+
+add_test(hello ${CMAKE_COMMAND} -E echo hello)
+add_test(goodbye ${CMAKE_COMMAND} -E echo goodbye)
diff --git
a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-ctest-stdout.txt
b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-ctest-stdout.txt
new file mode 100644
index 0000000000..d0a54872a9
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-ctest-stdout.txt
@@ -0,0 +1,15 @@
+^Test project .*/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-build
+ Start 1: initialization
+1/4 Test #1: initialization ................... Passed +[0-9.]+ sec
+ Start 2: test1
+2/4 Test #2: test1 ............................\*\*\*Timeout +[0-9.]+ sec
+ Start 2: test1
+ Test #2: test1 ............................ Passed +[0-9.]+ sec
+ Start 3: hello
+3/4 Test #3: hello ............................ Passed +[0-9.]+ sec
+ Start 4: goodbye
+4/4 Test #4: goodbye .......................... Passed +[0-9.]+ sec
+
+100% tests passed, 0 tests failed out of 4
+
+Total Test time \(real\) = +[0-9.]+ sec$
diff --git
a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-good-stderr.txt
b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-good-stderr.txt
new file mode 100644
index 0000000000..a7c4b11f76
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-good-stderr.txt
@@ -0,0 +1 @@
+^No tests were found!!!$
diff --git
a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-result.txt
b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-result.txt
new file mode 100644
index 0000000000..d00491fd7e
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-result.txt
@@ -0,0 +1 @@
+1
diff --git
a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-stderr.txt
b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-stderr.txt
new file mode 100644
index 0000000000..15ee3a98ea
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-stderr.txt
@@ -0,0 +1 @@
+^CMake Error: At most one '--repeat-\*' option may be used\.$
diff --git a/Tests/RunCMake/CTestCommandLine/test1-timeout.cmake
b/Tests/RunCMake/CTestCommandLine/test1-timeout.cmake
new file mode 100644
index 0000000000..fbf2cccd46
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/test1-timeout.cmake
@@ -0,0 +1,14 @@
+# This is run by test test1 in repeat-after-timeout-cmake.cmake with cmake -P.
+# It reads the file TEST_OUTPUT_FILE and increments the number
+# found in the file by 1. Unless the number is 2, then the
+# code sends out a cmake error causing the test to not timeout only on
+# the second time it is run.
+message("TEST_OUTPUT_FILE = ${TEST_OUTPUT_FILE}")
+file(READ "${TEST_OUTPUT_FILE}" COUNT)
+message("COUNT= ${COUNT}")
+math(EXPR COUNT "${COUNT} + 1")
+file(WRITE "${TEST_OUTPUT_FILE}" "${COUNT}")
+if(NOT COUNT EQUAL 2)
+ message("this test times out except on the 2nd run")
+ execute_process(COMMAND ${CMAKE_COMMAND} -E sleep 10)
+endif()
https://cmake.org/gitweb?p=cmake.git;a=commitdiff;h=80c2c9d14cf1c1a8f162e119bd00d5f483a94af2
commit 80c2c9d14cf1c1a8f162e119bd00d5f483a94af2
Author: Brad King <brad.k...@kitware.com>
AuthorDate: Tue Oct 29 14:45:21 2019 -0400
Commit: Brad King <brad.k...@kitware.com>
CommitDate: Tue Oct 29 15:10:12 2019 -0400
ctest: Add --repeat-until-pass option
Add an option to re-run tests if they fail. This will help tolerate
sporadic failures.
Issue: #17010
Co-Author: Ben Boeckel <ben.boec...@kitware.com>
Co-Author: Chuck Atkins <chuck.atk...@kitware.com>
diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst
index a18d43f48b..031b9a7636 100644
--- a/Help/manual/ctest.1.rst
+++ b/Help/manual/ctest.1.rst
@@ -266,6 +266,11 @@ Options
This is useful in finding sporadic failures in test cases.
+``--repeat-until-pass <n>``
+ Allow each test to run up to ``<n>`` times in order to pass.
+
+ This is useful in tolerating sporadic failures in test cases.
+
``--max-width <width>``
Set the max width for a test name to output.
diff --git a/Help/release/dev/ctest-repeat-until-pass.rst
b/Help/release/dev/ctest-repeat-until-pass.rst
new file mode 100644
index 0000000000..003cdf2aec
--- /dev/null
+++ b/Help/release/dev/ctest-repeat-until-pass.rst
@@ -0,0 +1,5 @@
+ctest-repeat-until-pass
+-----------------------
+
+* The :manual:`ctest(1)` tool learned a new ``--repeat-until-pass <n>``
+ option to help tolerate sporadic test failures.
diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx
b/Source/CTest/cmCTestMultiProcessHandler.cxx
index 7e8d5482db..4812f3036f 100644
--- a/Source/CTest/cmCTestMultiProcessHandler.cxx
+++ b/Source/CTest/cmCTestMultiProcessHandler.cxx
@@ -171,8 +171,8 @@ bool cmCTestMultiProcessHandler::StartTestProcess(int test)
this->RunningCount += GetProcessorsUsed(test);
cmCTestRunTest* testRun = new cmCTestRunTest(*this);
- if (this->CTest->GetRepeatUntilFail()) {
- testRun->SetRunUntilFailOn();
+ if (this->CTest->GetRerunMode() != cmCTest::Rerun::Never) {
+ testRun->SetRerunMode(this->CTest->GetRerunMode());
testRun->SetNumberOfRuns(this->CTest->GetTestRepeat());
}
testRun->SetIndex(test);
diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx
index 2e7836a10a..ce9e13bbba 100644
--- a/Source/CTest/cmCTestRunTest.cxx
+++ b/Source/CTest/cmCTestRunTest.cxx
@@ -340,10 +340,12 @@ bool cmCTestRunTest::NeedsToRerun()
return false;
}
// if number of runs left is not 0, and we are running until
- // we find a failed test, then return true so the test can be
+ // we find a failed (or passed) test, then return true so the test can be
// restarted
- if (this->RunUntilFail &&
- this->TestResult.Status == cmCTestTestHandler::COMPLETED) {
+ if ((this->RerunMode == cmCTest::Rerun::UntilFail &&
+ this->TestResult.Status == cmCTestTestHandler::COMPLETED) ||
+ (this->RerunMode == cmCTest::Rerun::UntilPass &&
+ this->TestResult.Status != cmCTestTestHandler::COMPLETED)) {
this->RunAgain = true;
return true;
}
@@ -743,7 +745,11 @@ void cmCTestRunTest::WriteLogOutputTop(size_t completed,
size_t total)
// then it will never print out the completed / total, same would
// got for run until pass. Trick is when this is called we don't
// yet know if we are passing or failing.
- if (this->NumberOfRunsLeft == 1 || this->CTest->GetTestProgressOutput()) {
+ if ((this->RerunMode != cmCTest::Rerun::UntilPass &&
+ this->NumberOfRunsLeft == 1) ||
+ (this->RerunMode == cmCTest::Rerun::UntilPass &&
+ this->NumberOfRunsLeft == this->NumberOfRunsTotal) ||
+ this->CTest->GetTestProgressOutput()) {
outputStream << std::setw(getNumWidth(total)) << completed << "/";
outputStream << std::setw(getNumWidth(total)) << total << " ";
}
diff --git a/Source/CTest/cmCTestRunTest.h b/Source/CTest/cmCTestRunTest.h
index 7b202d1e9f..881cbb6486 100644
--- a/Source/CTest/cmCTestRunTest.h
+++ b/Source/CTest/cmCTestRunTest.h
@@ -13,13 +13,12 @@
#include <stddef.h>
+#include "cmCTest.h"
#include "cmCTestMultiProcessHandler.h"
#include "cmCTestTestHandler.h"
#include "cmDuration.h"
#include "cmProcess.h"
-class cmCTest;
-
/** \class cmRunTest
* \brief represents a single test to be run
*
@@ -30,8 +29,13 @@ class cmCTestRunTest
public:
explicit cmCTestRunTest(cmCTestMultiProcessHandler& multiHandler);
- void SetNumberOfRuns(int n) { this->NumberOfRunsLeft = n; }
- void SetRunUntilFailOn() { this->RunUntilFail = true; }
+ void SetNumberOfRuns(int n)
+ {
+ this->NumberOfRunsLeft = n;
+ this->NumberOfRunsTotal = n;
+ }
+
+ void SetRerunMode(cmCTest::Rerun r) { this->RerunMode = r; }
void SetTestProperties(cmCTestTestHandler::cmCTestTestProperties* prop)
{
this->TestProperties = prop;
@@ -129,8 +133,9 @@ private:
std::vector<std::map<
std::string, std::vector<cmCTestMultiProcessHandler::HardwareAllocation>>>
AllocatedHardware;
- bool RunUntilFail = false; // default to run the test once
+ cmCTest::Rerun RerunMode = cmCTest::Rerun::Never;
int NumberOfRunsLeft = 1; // default to 1 run of the test
+ int NumberOfRunsTotal = 1; // default to 1 run of the test
bool RunAgain = false; // default to not having to run again
size_t TotalNumberOfTests;
};
diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx
index 10b7646664..7276d98fba 100644
--- a/Source/cmCTest.cxx
+++ b/Source/cmCTest.cxx
@@ -84,7 +84,7 @@ struct cmCTest::Private
};
int RepeatTests = 1; // default to run each test once
- bool RepeatUntilFail = false;
+ cmCTest::Rerun RerunMode = cmCTest::Rerun::Never;
std::string ConfigType;
std::string ScheduleType;
std::chrono::system_clock::time_point StopTime;
@@ -1839,11 +1839,16 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
this->SetParallelLevel(plevel);
this->Impl->ParallelLevelSetInCli = true;
}
+
if (this->CheckArgument(arg, "--repeat-until-fail")) {
if (i >= args.size() - 1) {
errormsg = "'--repeat-until-fail' requires an argument";
return false;
}
+ if (this->Impl->RerunMode != cmCTest::Rerun::Never) {
+ errormsg = "At most one '--repeat-*' option may be used.";
+ return false;
+ }
i++;
long repeat = 1;
if (!cmStrToLong(args[i], &repeat)) {
@@ -1853,7 +1858,29 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
}
this->Impl->RepeatTests = static_cast<int>(repeat);
if (repeat > 1) {
- this->Impl->RepeatUntilFail = true;
+ this->Impl->RerunMode = cmCTest::Rerun::UntilFail;
+ }
+ }
+
+ if (this->CheckArgument(arg, "--repeat-until-pass")) {
+ if (i >= args.size() - 1) {
+ errormsg = "'--repeat-until-pass' requires an argument";
+ return false;
+ }
+ if (this->Impl->RerunMode != cmCTest::Rerun::Never) {
+ errormsg = "At most one '--repeat-*' option may be used.";
+ return false;
+ }
+ i++;
+ long repeat = 1;
+ if (!cmStrToLong(args[i], &repeat)) {
+ errormsg =
+ "'--repeat-until-pass' given non-integer value '" + args[i] + "'";
+ return false;
+ }
+ this->Impl->RepeatTests = static_cast<int>(repeat);
+ if (repeat > 1) {
+ this->Impl->RerunMode = cmCTest::Rerun::UntilPass;
}
}
@@ -2852,9 +2879,9 @@ int cmCTest::GetTestRepeat() const
return this->Impl->RepeatTests;
}
-bool cmCTest::GetRepeatUntilFail() const
+cmCTest::Rerun cmCTest::GetRerunMode() const
{
- return this->Impl->RepeatUntilFail;
+ return this->Impl->RerunMode;
}
void cmCTest::SetBuildID(const std::string& id)
diff --git a/Source/cmCTest.h b/Source/cmCTest.h
index 82a6f4cbd3..c6b89281d7 100644
--- a/Source/cmCTest.h
+++ b/Source/cmCTest.h
@@ -433,8 +433,13 @@ public:
/** Return the number of times a test should be run */
int GetTestRepeat() const;
- /** Return true if test should run until fail */
- bool GetRepeatUntilFail() const;
+ enum class Rerun
+ {
+ Never,
+ UntilFail,
+ UntilPass,
+ };
+ Rerun GetRerunMode() const;
void GenerateSubprojectsOutput(cmXMLWriter& xml);
std::vector<std::string> GetLabelsForSubprojects();
diff --git a/Source/ctest.cxx b/Source/ctest.cxx
index 91ee598b25..f716d7a941 100644
--- a/Source/ctest.cxx
+++ b/Source/ctest.cxx
@@ -99,8 +99,9 @@ static const char* cmDocumentationOptions[][2] = {
{ "-U, --union", "Take the Union of -I and -R" },
{ "--rerun-failed", "Run only the tests that failed previously" },
{ "--repeat-until-fail <n>",
- "Require each test to run <n> "
- "times without failing in order to pass" },
+ "Require each test to run <n> times without failing in order to pass" },
+ { "--repeat-until-pass <n>",
+ "Allow each test to run up to <n> times in order to pass" },
{ "--max-width <width>", "Set the max width for a test name to output" },
{ "--interactive-debug-mode [0|1]", "Set the interactive mode to 0 or 1." },
{ "--hardware-spec-file <file>", "Set the hardware spec file to use." },
diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
index fd2c97fef0..15a90fc02f 100644
--- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
+++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
@@ -4,6 +4,16 @@ set(RunCMake_TEST_TIMEOUT 60)
unset(ENV{CTEST_PARALLEL_LEVEL})
unset(ENV{CTEST_OUTPUT_ON_FAILURE})
+run_cmake_command(repeat-until-pass-bad1
+ ${CMAKE_CTEST_COMMAND} --repeat-until-pass
+ )
+run_cmake_command(repeat-until-pass-bad2
+ ${CMAKE_CTEST_COMMAND} --repeat-until-pass foo
+ )
+run_cmake_command(repeat-until-pass-good
+ ${CMAKE_CTEST_COMMAND} --repeat-until-pass 2
+ )
+
run_cmake_command(repeat-until-fail-bad1
${CMAKE_CTEST_COMMAND} --repeat-until-fail
)
@@ -14,14 +24,29 @@ run_cmake_command(repeat-until-fail-good
${CMAKE_CTEST_COMMAND} --repeat-until-fail 2
)
-function(run_repeat_until_fail_tests)
+run_cmake_command(repeat-until-pass-and-fail
+ ${CMAKE_CTEST_COMMAND} --repeat-until-pass 2 --repeat-until-fail 2
+ )
+run_cmake_command(repeat-until-fail-and-pass
+ ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat-until-pass 2
+ )
+
+function(run_repeat_until_pass_tests)
# Use a single build tree for a few tests without cleaning.
- set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-fail-build)
+ set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-pass-build)
+ run_cmake(repeat-until-pass-cmake)
set(RunCMake_TEST_NO_CLEAN 1)
- file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
- file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
+ run_cmake_command(repeat-until-pass-ctest
+ ${CMAKE_CTEST_COMMAND} -C Debug --repeat-until-pass 3
+ )
+endfunction()
+run_repeat_until_pass_tests()
+function(run_repeat_until_fail_tests)
+ # Use a single build tree for a few tests without cleaning.
+ set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-fail-build)
run_cmake(repeat-until-fail-cmake)
+ set(RunCMake_TEST_NO_CLEAN 1)
run_cmake_command(repeat-until-fail-ctest
${CMAKE_CTEST_COMMAND} -C Debug --repeat-until-fail 3
)
diff --git
a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-result.txt
b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-result.txt
new file mode 100644
index 0000000000..d00491fd7e
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-result.txt
@@ -0,0 +1 @@
+1
diff --git
a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-stderr.txt
b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-stderr.txt
new file mode 100644
index 0000000000..15ee3a98ea
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-stderr.txt
@@ -0,0 +1 @@
+^CMake Error: At most one '--repeat-\*' option may be used\.$
diff --git
a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-result.txt
b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-result.txt
new file mode 100644
index 0000000000..d00491fd7e
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-result.txt
@@ -0,0 +1 @@
+1
diff --git
a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-stderr.txt
b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-stderr.txt
new file mode 100644
index 0000000000..15ee3a98ea
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-stderr.txt
@@ -0,0 +1 @@
+^CMake Error: At most one '--repeat-\*' option may be used\.$
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-result.txt
b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-result.txt
new file mode 100644
index 0000000000..d00491fd7e
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-stderr.txt
b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-stderr.txt
new file mode 100644
index 0000000000..c6afb1d9cf
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-stderr.txt
@@ -0,0 +1 @@
+^CMake Error: '--repeat-until-pass' requires an argument$
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-result.txt
b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-result.txt
new file mode 100644
index 0000000000..d00491fd7e
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-stderr.txt
b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-stderr.txt
new file mode 100644
index 0000000000..cc3aed5330
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-stderr.txt
@@ -0,0 +1 @@
+^CMake Error: '--repeat-until-pass' given non-integer value 'foo'$
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-cmake.cmake
b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-cmake.cmake
new file mode 100644
index 0000000000..d1095518f3
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-cmake.cmake
@@ -0,0 +1,15 @@
+enable_testing()
+
+set(TEST_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/test_output.txt")
+add_test(NAME initialization
+ COMMAND ${CMAKE_COMMAND}
+ "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
+ -P "${CMAKE_CURRENT_SOURCE_DIR}/init.cmake")
+add_test(NAME test1
+ COMMAND ${CMAKE_COMMAND}
+ "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
+ -P "${CMAKE_CURRENT_SOURCE_DIR}/test1-pass.cmake")
+set_tests_properties(test1 PROPERTIES DEPENDS "initialization")
+
+add_test(hello ${CMAKE_COMMAND} -E echo hello)
+add_test(goodbye ${CMAKE_COMMAND} -E echo goodbye)
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-ctest-stdout.txt
b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-ctest-stdout.txt
new file mode 100644
index 0000000000..3745dc252b
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-ctest-stdout.txt
@@ -0,0 +1,15 @@
+^Test project .*/Tests/RunCMake/CTestCommandLine/repeat-until-pass-build
+ Start 1: initialization
+1/4 Test #1: initialization ................... Passed +[0-9.]+ sec
+ Start 2: test1
+2/4 Test #2: test1 ............................\*\*\*Failed +[0-9.]+ sec
+ Start 2: test1
+ Test #2: test1 ............................ Passed +[0-9.]+ sec
+ Start 3: hello
+3/4 Test #3: hello ............................ Passed +[0-9.]+ sec
+ Start 4: goodbye
+4/4 Test #4: goodbye .......................... Passed +[0-9.]+ sec
+
+100% tests passed, 0 tests failed out of 4
+
+Total Test time \(real\) = +[0-9.]+ sec$
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-good-stderr.txt
b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-good-stderr.txt
new file mode 100644
index 0000000000..a7c4b11f76
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-good-stderr.txt
@@ -0,0 +1 @@
+^No tests were found!!!$
diff --git a/Tests/RunCMake/CTestCommandLine/test1-pass.cmake
b/Tests/RunCMake/CTestCommandLine/test1-pass.cmake
new file mode 100644
index 0000000000..dda8deae50
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/test1-pass.cmake
@@ -0,0 +1,13 @@
+# This is run by test test1 in repeat-until-pass-cmake.cmake with cmake -P.
+# It reads the file TEST_OUTPUT_FILE and increments the number
+# found in the file by 1. Unless the number is 2, then the
+# code sends out a cmake error causing the test to pass only on
+# the second time it is run.
+message("TEST_OUTPUT_FILE = ${TEST_OUTPUT_FILE}")
+file(READ "${TEST_OUTPUT_FILE}" COUNT)
+message("COUNT= ${COUNT}")
+math(EXPR COUNT "${COUNT} + 1")
+file(WRITE "${TEST_OUTPUT_FILE}" "${COUNT}")
+if(NOT COUNT EQUAL 2)
+ message(FATAL_ERROR "this test passes only on the 2nd run")
+endif()
https://cmake.org/gitweb?p=cmake.git;a=commitdiff;h=0187e522448c8fe92c15d4a983afac36950d1d59
commit 0187e522448c8fe92c15d4a983afac36950d1d59
Author: Brad King <brad.k...@kitware.com>
AuthorDate: Tue Oct 29 13:41:52 2019 -0400
Commit: Brad King <brad.k...@kitware.com>
CommitDate: Tue Oct 29 13:47:23 2019 -0400
cmCTestRunTest: Use inline member initializers
diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx
index 7f7f736799..2e7836a10a 100644
--- a/Source/CTest/cmCTestRunTest.cxx
+++ b/Source/CTest/cmCTestRunTest.cxx
@@ -34,9 +34,6 @@ cmCTestRunTest::cmCTestRunTest(cmCTestMultiProcessHandler&
multiHandler)
this->TestResult.Status = cmCTestTestHandler::NOT_RUN;
this->TestResult.TestCount = 0;
this->TestResult.Properties = nullptr;
- this->NumberOfRunsLeft = 1; // default to 1 run of the test
- this->RunUntilFail = false; // default to run the test once
- this->RunAgain = false; // default to not having to run again
}
void cmCTestRunTest::CheckOutput(std::string const& line)
diff --git a/Source/CTest/cmCTestRunTest.h b/Source/CTest/cmCTestRunTest.h
index 085a6b8f84..7b202d1e9f 100644
--- a/Source/CTest/cmCTestRunTest.h
+++ b/Source/CTest/cmCTestRunTest.h
@@ -129,9 +129,9 @@ private:
std::vector<std::map<
std::string, std::vector<cmCTestMultiProcessHandler::HardwareAllocation>>>
AllocatedHardware;
- bool RunUntilFail;
- int NumberOfRunsLeft;
- bool RunAgain;
+ bool RunUntilFail = false; // default to run the test once
+ int NumberOfRunsLeft = 1; // default to 1 run of the test
+ bool RunAgain = false; // default to not having to run again
size_t TotalNumberOfTests;
};
-----------------------------------------------------------------------
Summary of changes:
Help/manual/ctest.1.rst | 13 +++++
Help/release/dev/ctest-repeat-until-pass.rst | 6 +++
Source/CTest/cmCTestMultiProcessHandler.cxx | 4 +-
Source/CTest/cmCTestRunTest.cxx | 20 +++++---
Source/CTest/cmCTestRunTest.h | 19 +++++---
Source/cmCTest.cxx | 57 ++++++++++++++++++++--
Source/cmCTest.h | 10 +++-
Source/ctest.cxx | 7 ++-
Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake | 57 ++++++++++++++++++++--
.../repeat-after-timeout-bad1-result.txt} | 0
.../repeat-after-timeout-bad1-stderr.txt | 1 +
.../repeat-after-timeout-bad2-result.txt} | 0
.../repeat-after-timeout-bad2-stderr.txt | 1 +
.../repeat-after-timeout-cmake.cmake | 15 ++++++
.../repeat-after-timeout-ctest-stdout.txt | 15 ++++++
...rr.txt => repeat-after-timeout-good-stderr.txt} | 0
.../repeat-until-fail-and-pass-result.txt} | 0
.../repeat-until-fail-and-pass-stderr.txt | 1 +
.../repeat-until-fail-and-timeout-result.txt} | 0
.../repeat-until-fail-and-timeout-stderr.txt | 1 +
.../repeat-until-pass-and-fail-result.txt} | 0
.../repeat-until-pass-and-fail-stderr.txt | 1 +
.../repeat-until-pass-bad1-result.txt} | 0
.../repeat-until-pass-bad1-stderr.txt | 1 +
.../repeat-until-pass-bad2-result.txt} | 0
.../repeat-until-pass-bad2-stderr.txt | 1 +
.../CTestCommandLine/repeat-until-pass-cmake.cmake | 15 ++++++
.../repeat-until-pass-ctest-stdout.txt | 15 ++++++
...tderr.txt => repeat-until-pass-good-stderr.txt} | 0
Tests/RunCMake/CTestCommandLine/test1-pass.cmake | 13 +++++
.../RunCMake/CTestCommandLine/test1-timeout.cmake | 14 ++++++
31 files changed, 259 insertions(+), 28 deletions(-)
create mode 100644 Help/release/dev/ctest-repeat-until-pass.rst
copy Tests/RunCMake/{Android/BadSYSROOT-result.txt =>
CTestCommandLine/repeat-after-timeout-bad1-result.txt} (100%)
create mode 100644
Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-stderr.txt
copy Tests/RunCMake/{Android/BadSYSROOT-result.txt =>
CTestCommandLine/repeat-after-timeout-bad2-result.txt} (100%)
create mode 100644
Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-stderr.txt
create mode 100644
Tests/RunCMake/CTestCommandLine/repeat-after-timeout-cmake.cmake
create mode 100644
Tests/RunCMake/CTestCommandLine/repeat-after-timeout-ctest-stdout.txt
copy Tests/RunCMake/CTestCommandLine/{repeat-until-fail-good-stderr.txt =>
repeat-after-timeout-good-stderr.txt} (100%)
copy Tests/RunCMake/{Android/BadSYSROOT-result.txt =>
CTestCommandLine/repeat-until-fail-and-pass-result.txt} (100%)
create mode 100644
Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-stderr.txt
copy Tests/RunCMake/{Android/BadSYSROOT-result.txt =>
CTestCommandLine/repeat-until-fail-and-timeout-result.txt} (100%)
create mode 100644
Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-stderr.txt
copy Tests/RunCMake/{Android/BadSYSROOT-result.txt =>
CTestCommandLine/repeat-until-pass-and-fail-result.txt} (100%)
create mode 100644
Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-stderr.txt
copy Tests/RunCMake/{Android/BadSYSROOT-result.txt =>
CTestCommandLine/repeat-until-pass-bad1-result.txt} (100%)
create mode 100644
Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-stderr.txt
copy Tests/RunCMake/{Android/BadSYSROOT-result.txt =>
CTestCommandLine/repeat-until-pass-bad2-result.txt} (100%)
create mode 100644
Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-stderr.txt
create mode 100644
Tests/RunCMake/CTestCommandLine/repeat-until-pass-cmake.cmake
create mode 100644
Tests/RunCMake/CTestCommandLine/repeat-until-pass-ctest-stdout.txt
copy Tests/RunCMake/CTestCommandLine/{repeat-until-fail-good-stderr.txt =>
repeat-until-pass-good-stderr.txt} (100%)
create mode 100644 Tests/RunCMake/CTestCommandLine/test1-pass.cmake
create mode 100644 Tests/RunCMake/CTestCommandLine/test1-timeout.cmake
hooks/post-receive
--
CMake
_______________________________________________
Cmake-commits mailing list
Cmake-commits@cmake.org
https://cmake.org/mailman/listinfo/cmake-commits
