This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "CMake".
The branch, master has been updated via 6660926f22b940759954e74e2b4c0c3dd0ff1991 (commit) via 39ac8b4eb5c5ea99cf1053bd37e76d811f5122fc (commit) via 80c2c9d14cf1c1a8f162e119bd00d5f483a94af2 (commit) via 0187e522448c8fe92c15d4a983afac36950d1d59 (commit) from b42da462333aea986576d00bf9af7449aa6e8010 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- https://cmake.org/gitweb?p=cmake.git;a=commitdiff;h=6660926f22b940759954e74e2b4c0c3dd0ff1991 commit 6660926f22b940759954e74e2b4c0c3dd0ff1991 Merge: b42da46233 39ac8b4eb5 Author: Brad King <brad.k...@kitware.com> AuthorDate: Fri Nov 1 13:55:09 2019 +0000 Commit: Kitware Robot <kwro...@kitware.com> CommitDate: Fri Nov 1 09:55:18 2019 -0400 Merge topic 'ctest-repeat-until-pass' 39ac8b4eb5 ctest: Add --repeat-after-timeout option 80c2c9d14c ctest: Add --repeat-until-pass option 0187e52244 cmCTestRunTest: Use inline member initializers Acked-by: Kitware Robot <kwro...@kitware.com> Merge-request: !3960 https://cmake.org/gitweb?p=cmake.git;a=commitdiff;h=39ac8b4eb5c5ea99cf1053bd37e76d811f5122fc commit 39ac8b4eb5c5ea99cf1053bd37e76d811f5122fc Author: Brad King <brad.k...@kitware.com> AuthorDate: Tue Oct 29 14:21:38 2019 -0400 Commit: Brad King <brad.k...@kitware.com> CommitDate: Tue Oct 29 15:14:36 2019 -0400 ctest: Add --repeat-after-timeout option Add an option to re-run tests if they timeout. This will help tolerate sporadic timeouts on busy machines. diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst index 031b9a7636..0097cee148 100644 --- a/Help/manual/ctest.1.rst +++ b/Help/manual/ctest.1.rst @@ -268,9 +268,17 @@ Options ``--repeat-until-pass <n>`` Allow each test to run up to ``<n>`` times in order to pass. + Repeats tests if they fail for any reason. This is useful in tolerating sporadic failures in test cases. +``--repeat-after-timeout <n>`` + Allow each test to run up to ``<n>`` times in order to pass. + Repeats tests only if they timeout. + + This is useful in tolerating sporadic timeouts in test cases + on busy machines. + ``--max-width <width>`` Set the max width for a test name to output. diff --git a/Help/release/dev/ctest-repeat-until-pass.rst b/Help/release/dev/ctest-repeat-until-pass.rst index 003cdf2aec..d177247181 100644 --- a/Help/release/dev/ctest-repeat-until-pass.rst +++ b/Help/release/dev/ctest-repeat-until-pass.rst @@ -1,5 +1,6 @@ ctest-repeat-until-pass ----------------------- -* The :manual:`ctest(1)` tool learned a new ``--repeat-until-pass <n>`` - option to help tolerate sporadic test failures. +* The :manual:`ctest(1)` tool learned new ``--repeat-until-pass <n>`` + and ``--repeat-after-timeout <n>`` options to help tolerate sporadic + test failures. diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx index ce9e13bbba..ba146532b6 100644 --- a/Source/CTest/cmCTestRunTest.cxx +++ b/Source/CTest/cmCTestRunTest.cxx @@ -345,7 +345,9 @@ bool cmCTestRunTest::NeedsToRerun() if ((this->RerunMode == cmCTest::Rerun::UntilFail && this->TestResult.Status == cmCTestTestHandler::COMPLETED) || (this->RerunMode == cmCTest::Rerun::UntilPass && - this->TestResult.Status != cmCTestTestHandler::COMPLETED)) { + this->TestResult.Status != cmCTestTestHandler::COMPLETED) || + (this->RerunMode == cmCTest::Rerun::AfterTimeout && + this->TestResult.Status == cmCTestTestHandler::TIMEOUT)) { this->RunAgain = true; return true; } @@ -745,10 +747,11 @@ void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total) // then it will never print out the completed / total, same would // got for run until pass. Trick is when this is called we don't // yet know if we are passing or failing. - if ((this->RerunMode != cmCTest::Rerun::UntilPass && - this->NumberOfRunsLeft == 1) || - (this->RerunMode == cmCTest::Rerun::UntilPass && - this->NumberOfRunsLeft == this->NumberOfRunsTotal) || + bool const progressOnLast = + (this->RerunMode != cmCTest::Rerun::UntilPass && + this->RerunMode != cmCTest::Rerun::AfterTimeout); + if ((progressOnLast && this->NumberOfRunsLeft == 1) || + (!progressOnLast && this->NumberOfRunsLeft == this->NumberOfRunsTotal) || this->CTest->GetTestProgressOutput()) { outputStream << std::setw(getNumWidth(total)) << completed << "/"; outputStream << std::setw(getNumWidth(total)) << total << " "; diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index 7276d98fba..20445b0853 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -1884,6 +1884,28 @@ bool cmCTest::HandleCommandLineArguments(size_t& i, } } + if (this->CheckArgument(arg, "--repeat-after-timeout")) { + if (i >= args.size() - 1) { + errormsg = "'--repeat-after-timeout' requires an argument"; + return false; + } + if (this->Impl->RerunMode != cmCTest::Rerun::Never) { + errormsg = "At most one '--repeat-*' option may be used."; + return false; + } + i++; + long repeat = 1; + if (!cmStrToLong(args[i], &repeat)) { + errormsg = + "'--repeat-after-timeout' given non-integer value '" + args[i] + "'"; + return false; + } + this->Impl->RepeatTests = static_cast<int>(repeat); + if (repeat > 1) { + this->Impl->RerunMode = cmCTest::Rerun::AfterTimeout; + } + } + if (this->CheckArgument(arg, "--test-load") && i < args.size() - 1) { i++; unsigned long load; diff --git a/Source/cmCTest.h b/Source/cmCTest.h index c6b89281d7..bef0f8d368 100644 --- a/Source/cmCTest.h +++ b/Source/cmCTest.h @@ -438,6 +438,7 @@ public: Never, UntilFail, UntilPass, + AfterTimeout, }; Rerun GetRerunMode() const; diff --git a/Source/ctest.cxx b/Source/ctest.cxx index f716d7a941..2659e3071b 100644 --- a/Source/ctest.cxx +++ b/Source/ctest.cxx @@ -102,6 +102,8 @@ static const char* cmDocumentationOptions[][2] = { "Require each test to run <n> times without failing in order to pass" }, { "--repeat-until-pass <n>", "Allow each test to run up to <n> times in order to pass" }, + { "--repeat-after-timeout <n>", + "Allow each test to run up to <n> times if it times out" }, { "--max-width <width>", "Set the max width for a test name to output" }, { "--interactive-debug-mode [0|1]", "Set the interactive mode to 0 or 1." }, { "--hardware-spec-file <file>", "Set the hardware spec file to use." }, diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake index 15a90fc02f..d7f41336e8 100644 --- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake +++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake @@ -24,12 +24,25 @@ run_cmake_command(repeat-until-fail-good ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 ) +run_cmake_command(repeat-after-timeout-bad1 + ${CMAKE_CTEST_COMMAND} --repeat-after-timeout + ) +run_cmake_command(repeat-after-timeout-bad2 + ${CMAKE_CTEST_COMMAND} --repeat-after-timeout foo + ) +run_cmake_command(repeat-after-timeout-good + ${CMAKE_CTEST_COMMAND} --repeat-after-timeout 2 + ) + run_cmake_command(repeat-until-pass-and-fail ${CMAKE_CTEST_COMMAND} --repeat-until-pass 2 --repeat-until-fail 2 ) run_cmake_command(repeat-until-fail-and-pass ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat-until-pass 2 ) +run_cmake_command(repeat-until-fail-and-timeout + ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat-after-timeout 2 + ) function(run_repeat_until_pass_tests) # Use a single build tree for a few tests without cleaning. @@ -42,6 +55,17 @@ function(run_repeat_until_pass_tests) endfunction() run_repeat_until_pass_tests() +function(run_repeat_after_timeout_tests) + # Use a single build tree for a few tests without cleaning. + set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-after-timeout-build) + run_cmake(repeat-after-timeout-cmake) + set(RunCMake_TEST_NO_CLEAN 1) + run_cmake_command(repeat-after-timeout-ctest + ${CMAKE_CTEST_COMMAND} -C Debug --repeat-after-timeout 3 + ) +endfunction() +run_repeat_after_timeout_tests() + function(run_repeat_until_fail_tests) # Use a single build tree for a few tests without cleaning. set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-fail-build) diff --git a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-result.txt b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-result.txt new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-stderr.txt new file mode 100644 index 0000000000..aea92b8bb0 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-stderr.txt @@ -0,0 +1 @@ +^CMake Error: '--repeat-after-timeout' requires an argument$ diff --git a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-result.txt b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-result.txt new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-stderr.txt new file mode 100644 index 0000000000..c5db55b65b --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-stderr.txt @@ -0,0 +1 @@ +^CMake Error: '--repeat-after-timeout' given non-integer value 'foo'$ diff --git a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-cmake.cmake b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-cmake.cmake new file mode 100644 index 0000000000..4dc9c3648f --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-cmake.cmake @@ -0,0 +1,15 @@ +enable_testing() + +set(TEST_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/test_output.txt") +add_test(NAME initialization + COMMAND ${CMAKE_COMMAND} + "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/init.cmake") +add_test(NAME test1 + COMMAND ${CMAKE_COMMAND} + "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/test1-timeout.cmake") +set_tests_properties(test1 PROPERTIES DEPENDS "initialization" TIMEOUT 2) + +add_test(hello ${CMAKE_COMMAND} -E echo hello) +add_test(goodbye ${CMAKE_COMMAND} -E echo goodbye) diff --git a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-ctest-stdout.txt b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-ctest-stdout.txt new file mode 100644 index 0000000000..d0a54872a9 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-ctest-stdout.txt @@ -0,0 +1,15 @@ +^Test project .*/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-build + Start 1: initialization +1/4 Test #1: initialization ................... Passed +[0-9.]+ sec + Start 2: test1 +2/4 Test #2: test1 ............................\*\*\*Timeout +[0-9.]+ sec + Start 2: test1 + Test #2: test1 ............................ Passed +[0-9.]+ sec + Start 3: hello +3/4 Test #3: hello ............................ Passed +[0-9.]+ sec + Start 4: goodbye +4/4 Test #4: goodbye .......................... Passed +[0-9.]+ sec + +100% tests passed, 0 tests failed out of 4 + +Total Test time \(real\) = +[0-9.]+ sec$ diff --git a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-good-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-good-stderr.txt new file mode 100644 index 0000000000..a7c4b11f76 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-good-stderr.txt @@ -0,0 +1 @@ +^No tests were found!!!$ diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-result.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-result.txt new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-stderr.txt new file mode 100644 index 0000000000..15ee3a98ea --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-stderr.txt @@ -0,0 +1 @@ +^CMake Error: At most one '--repeat-\*' option may be used\.$ diff --git a/Tests/RunCMake/CTestCommandLine/test1-timeout.cmake b/Tests/RunCMake/CTestCommandLine/test1-timeout.cmake new file mode 100644 index 0000000000..fbf2cccd46 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/test1-timeout.cmake @@ -0,0 +1,14 @@ +# This is run by test test1 in repeat-after-timeout-cmake.cmake with cmake -P. +# It reads the file TEST_OUTPUT_FILE and increments the number +# found in the file by 1. Unless the number is 2, then the +# code sends out a cmake error causing the test to not timeout only on +# the second time it is run. +message("TEST_OUTPUT_FILE = ${TEST_OUTPUT_FILE}") +file(READ "${TEST_OUTPUT_FILE}" COUNT) +message("COUNT= ${COUNT}") +math(EXPR COUNT "${COUNT} + 1") +file(WRITE "${TEST_OUTPUT_FILE}" "${COUNT}") +if(NOT COUNT EQUAL 2) + message("this test times out except on the 2nd run") + execute_process(COMMAND ${CMAKE_COMMAND} -E sleep 10) +endif() https://cmake.org/gitweb?p=cmake.git;a=commitdiff;h=80c2c9d14cf1c1a8f162e119bd00d5f483a94af2 commit 80c2c9d14cf1c1a8f162e119bd00d5f483a94af2 Author: Brad King <brad.k...@kitware.com> AuthorDate: Tue Oct 29 14:45:21 2019 -0400 Commit: Brad King <brad.k...@kitware.com> CommitDate: Tue Oct 29 15:10:12 2019 -0400 ctest: Add --repeat-until-pass option Add an option to re-run tests if they fail. This will help tolerate sporadic failures. Issue: #17010 Co-Author: Ben Boeckel <ben.boec...@kitware.com> Co-Author: Chuck Atkins <chuck.atk...@kitware.com> diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst index a18d43f48b..031b9a7636 100644 --- a/Help/manual/ctest.1.rst +++ b/Help/manual/ctest.1.rst @@ -266,6 +266,11 @@ Options This is useful in finding sporadic failures in test cases. +``--repeat-until-pass <n>`` + Allow each test to run up to ``<n>`` times in order to pass. + + This is useful in tolerating sporadic failures in test cases. + ``--max-width <width>`` Set the max width for a test name to output. diff --git a/Help/release/dev/ctest-repeat-until-pass.rst b/Help/release/dev/ctest-repeat-until-pass.rst new file mode 100644 index 0000000000..003cdf2aec --- /dev/null +++ b/Help/release/dev/ctest-repeat-until-pass.rst @@ -0,0 +1,5 @@ +ctest-repeat-until-pass +----------------------- + +* The :manual:`ctest(1)` tool learned a new ``--repeat-until-pass <n>`` + option to help tolerate sporadic test failures. diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx index 7e8d5482db..4812f3036f 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.cxx +++ b/Source/CTest/cmCTestMultiProcessHandler.cxx @@ -171,8 +171,8 @@ bool cmCTestMultiProcessHandler::StartTestProcess(int test) this->RunningCount += GetProcessorsUsed(test); cmCTestRunTest* testRun = new cmCTestRunTest(*this); - if (this->CTest->GetRepeatUntilFail()) { - testRun->SetRunUntilFailOn(); + if (this->CTest->GetRerunMode() != cmCTest::Rerun::Never) { + testRun->SetRerunMode(this->CTest->GetRerunMode()); testRun->SetNumberOfRuns(this->CTest->GetTestRepeat()); } testRun->SetIndex(test); diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx index 2e7836a10a..ce9e13bbba 100644 --- a/Source/CTest/cmCTestRunTest.cxx +++ b/Source/CTest/cmCTestRunTest.cxx @@ -340,10 +340,12 @@ bool cmCTestRunTest::NeedsToRerun() return false; } // if number of runs left is not 0, and we are running until - // we find a failed test, then return true so the test can be + // we find a failed (or passed) test, then return true so the test can be // restarted - if (this->RunUntilFail && - this->TestResult.Status == cmCTestTestHandler::COMPLETED) { + if ((this->RerunMode == cmCTest::Rerun::UntilFail && + this->TestResult.Status == cmCTestTestHandler::COMPLETED) || + (this->RerunMode == cmCTest::Rerun::UntilPass && + this->TestResult.Status != cmCTestTestHandler::COMPLETED)) { this->RunAgain = true; return true; } @@ -743,7 +745,11 @@ void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total) // then it will never print out the completed / total, same would // got for run until pass. Trick is when this is called we don't // yet know if we are passing or failing. - if (this->NumberOfRunsLeft == 1 || this->CTest->GetTestProgressOutput()) { + if ((this->RerunMode != cmCTest::Rerun::UntilPass && + this->NumberOfRunsLeft == 1) || + (this->RerunMode == cmCTest::Rerun::UntilPass && + this->NumberOfRunsLeft == this->NumberOfRunsTotal) || + this->CTest->GetTestProgressOutput()) { outputStream << std::setw(getNumWidth(total)) << completed << "/"; outputStream << std::setw(getNumWidth(total)) << total << " "; } diff --git a/Source/CTest/cmCTestRunTest.h b/Source/CTest/cmCTestRunTest.h index 7b202d1e9f..881cbb6486 100644 --- a/Source/CTest/cmCTestRunTest.h +++ b/Source/CTest/cmCTestRunTest.h @@ -13,13 +13,12 @@ #include <stddef.h> +#include "cmCTest.h" #include "cmCTestMultiProcessHandler.h" #include "cmCTestTestHandler.h" #include "cmDuration.h" #include "cmProcess.h" -class cmCTest; - /** \class cmRunTest * \brief represents a single test to be run * @@ -30,8 +29,13 @@ class cmCTestRunTest public: explicit cmCTestRunTest(cmCTestMultiProcessHandler& multiHandler); - void SetNumberOfRuns(int n) { this->NumberOfRunsLeft = n; } - void SetRunUntilFailOn() { this->RunUntilFail = true; } + void SetNumberOfRuns(int n) + { + this->NumberOfRunsLeft = n; + this->NumberOfRunsTotal = n; + } + + void SetRerunMode(cmCTest::Rerun r) { this->RerunMode = r; } void SetTestProperties(cmCTestTestHandler::cmCTestTestProperties* prop) { this->TestProperties = prop; @@ -129,8 +133,9 @@ private: std::vector<std::map< std::string, std::vector<cmCTestMultiProcessHandler::HardwareAllocation>>> AllocatedHardware; - bool RunUntilFail = false; // default to run the test once + cmCTest::Rerun RerunMode = cmCTest::Rerun::Never; int NumberOfRunsLeft = 1; // default to 1 run of the test + int NumberOfRunsTotal = 1; // default to 1 run of the test bool RunAgain = false; // default to not having to run again size_t TotalNumberOfTests; }; diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index 10b7646664..7276d98fba 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -84,7 +84,7 @@ struct cmCTest::Private }; int RepeatTests = 1; // default to run each test once - bool RepeatUntilFail = false; + cmCTest::Rerun RerunMode = cmCTest::Rerun::Never; std::string ConfigType; std::string ScheduleType; std::chrono::system_clock::time_point StopTime; @@ -1839,11 +1839,16 @@ bool cmCTest::HandleCommandLineArguments(size_t& i, this->SetParallelLevel(plevel); this->Impl->ParallelLevelSetInCli = true; } + if (this->CheckArgument(arg, "--repeat-until-fail")) { if (i >= args.size() - 1) { errormsg = "'--repeat-until-fail' requires an argument"; return false; } + if (this->Impl->RerunMode != cmCTest::Rerun::Never) { + errormsg = "At most one '--repeat-*' option may be used."; + return false; + } i++; long repeat = 1; if (!cmStrToLong(args[i], &repeat)) { @@ -1853,7 +1858,29 @@ bool cmCTest::HandleCommandLineArguments(size_t& i, } this->Impl->RepeatTests = static_cast<int>(repeat); if (repeat > 1) { - this->Impl->RepeatUntilFail = true; + this->Impl->RerunMode = cmCTest::Rerun::UntilFail; + } + } + + if (this->CheckArgument(arg, "--repeat-until-pass")) { + if (i >= args.size() - 1) { + errormsg = "'--repeat-until-pass' requires an argument"; + return false; + } + if (this->Impl->RerunMode != cmCTest::Rerun::Never) { + errormsg = "At most one '--repeat-*' option may be used."; + return false; + } + i++; + long repeat = 1; + if (!cmStrToLong(args[i], &repeat)) { + errormsg = + "'--repeat-until-pass' given non-integer value '" + args[i] + "'"; + return false; + } + this->Impl->RepeatTests = static_cast<int>(repeat); + if (repeat > 1) { + this->Impl->RerunMode = cmCTest::Rerun::UntilPass; } } @@ -2852,9 +2879,9 @@ int cmCTest::GetTestRepeat() const return this->Impl->RepeatTests; } -bool cmCTest::GetRepeatUntilFail() const +cmCTest::Rerun cmCTest::GetRerunMode() const { - return this->Impl->RepeatUntilFail; + return this->Impl->RerunMode; } void cmCTest::SetBuildID(const std::string& id) diff --git a/Source/cmCTest.h b/Source/cmCTest.h index 82a6f4cbd3..c6b89281d7 100644 --- a/Source/cmCTest.h +++ b/Source/cmCTest.h @@ -433,8 +433,13 @@ public: /** Return the number of times a test should be run */ int GetTestRepeat() const; - /** Return true if test should run until fail */ - bool GetRepeatUntilFail() const; + enum class Rerun + { + Never, + UntilFail, + UntilPass, + }; + Rerun GetRerunMode() const; void GenerateSubprojectsOutput(cmXMLWriter& xml); std::vector<std::string> GetLabelsForSubprojects(); diff --git a/Source/ctest.cxx b/Source/ctest.cxx index 91ee598b25..f716d7a941 100644 --- a/Source/ctest.cxx +++ b/Source/ctest.cxx @@ -99,8 +99,9 @@ static const char* cmDocumentationOptions[][2] = { { "-U, --union", "Take the Union of -I and -R" }, { "--rerun-failed", "Run only the tests that failed previously" }, { "--repeat-until-fail <n>", - "Require each test to run <n> " - "times without failing in order to pass" }, + "Require each test to run <n> times without failing in order to pass" }, + { "--repeat-until-pass <n>", + "Allow each test to run up to <n> times in order to pass" }, { "--max-width <width>", "Set the max width for a test name to output" }, { "--interactive-debug-mode [0|1]", "Set the interactive mode to 0 or 1." }, { "--hardware-spec-file <file>", "Set the hardware spec file to use." }, diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake index fd2c97fef0..15a90fc02f 100644 --- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake +++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake @@ -4,6 +4,16 @@ set(RunCMake_TEST_TIMEOUT 60) unset(ENV{CTEST_PARALLEL_LEVEL}) unset(ENV{CTEST_OUTPUT_ON_FAILURE}) +run_cmake_command(repeat-until-pass-bad1 + ${CMAKE_CTEST_COMMAND} --repeat-until-pass + ) +run_cmake_command(repeat-until-pass-bad2 + ${CMAKE_CTEST_COMMAND} --repeat-until-pass foo + ) +run_cmake_command(repeat-until-pass-good + ${CMAKE_CTEST_COMMAND} --repeat-until-pass 2 + ) + run_cmake_command(repeat-until-fail-bad1 ${CMAKE_CTEST_COMMAND} --repeat-until-fail ) @@ -14,14 +24,29 @@ run_cmake_command(repeat-until-fail-good ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 ) -function(run_repeat_until_fail_tests) +run_cmake_command(repeat-until-pass-and-fail + ${CMAKE_CTEST_COMMAND} --repeat-until-pass 2 --repeat-until-fail 2 + ) +run_cmake_command(repeat-until-fail-and-pass + ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat-until-pass 2 + ) + +function(run_repeat_until_pass_tests) # Use a single build tree for a few tests without cleaning. - set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-fail-build) + set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-pass-build) + run_cmake(repeat-until-pass-cmake) set(RunCMake_TEST_NO_CLEAN 1) - file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}") - file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}") + run_cmake_command(repeat-until-pass-ctest + ${CMAKE_CTEST_COMMAND} -C Debug --repeat-until-pass 3 + ) +endfunction() +run_repeat_until_pass_tests() +function(run_repeat_until_fail_tests) + # Use a single build tree for a few tests without cleaning. + set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-fail-build) run_cmake(repeat-until-fail-cmake) + set(RunCMake_TEST_NO_CLEAN 1) run_cmake_command(repeat-until-fail-ctest ${CMAKE_CTEST_COMMAND} -C Debug --repeat-until-fail 3 ) diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-result.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-result.txt new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-stderr.txt new file mode 100644 index 0000000000..15ee3a98ea --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-stderr.txt @@ -0,0 +1 @@ +^CMake Error: At most one '--repeat-\*' option may be used\.$ diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-result.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-result.txt new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-stderr.txt new file mode 100644 index 0000000000..15ee3a98ea --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-stderr.txt @@ -0,0 +1 @@ +^CMake Error: At most one '--repeat-\*' option may be used\.$ diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-result.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-result.txt new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-stderr.txt new file mode 100644 index 0000000000..c6afb1d9cf --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-stderr.txt @@ -0,0 +1 @@ +^CMake Error: '--repeat-until-pass' requires an argument$ diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-result.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-result.txt new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-stderr.txt new file mode 100644 index 0000000000..cc3aed5330 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-stderr.txt @@ -0,0 +1 @@ +^CMake Error: '--repeat-until-pass' given non-integer value 'foo'$ diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-cmake.cmake b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-cmake.cmake new file mode 100644 index 0000000000..d1095518f3 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-cmake.cmake @@ -0,0 +1,15 @@ +enable_testing() + +set(TEST_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/test_output.txt") +add_test(NAME initialization + COMMAND ${CMAKE_COMMAND} + "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/init.cmake") +add_test(NAME test1 + COMMAND ${CMAKE_COMMAND} + "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/test1-pass.cmake") +set_tests_properties(test1 PROPERTIES DEPENDS "initialization") + +add_test(hello ${CMAKE_COMMAND} -E echo hello) +add_test(goodbye ${CMAKE_COMMAND} -E echo goodbye) diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-ctest-stdout.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-ctest-stdout.txt new file mode 100644 index 0000000000..3745dc252b --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-ctest-stdout.txt @@ -0,0 +1,15 @@ +^Test project .*/Tests/RunCMake/CTestCommandLine/repeat-until-pass-build + Start 1: initialization +1/4 Test #1: initialization ................... Passed +[0-9.]+ sec + Start 2: test1 +2/4 Test #2: test1 ............................\*\*\*Failed +[0-9.]+ sec + Start 2: test1 + Test #2: test1 ............................ Passed +[0-9.]+ sec + Start 3: hello +3/4 Test #3: hello ............................ Passed +[0-9.]+ sec + Start 4: goodbye +4/4 Test #4: goodbye .......................... Passed +[0-9.]+ sec + +100% tests passed, 0 tests failed out of 4 + +Total Test time \(real\) = +[0-9.]+ sec$ diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-good-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-good-stderr.txt new file mode 100644 index 0000000000..a7c4b11f76 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-good-stderr.txt @@ -0,0 +1 @@ +^No tests were found!!!$ diff --git a/Tests/RunCMake/CTestCommandLine/test1-pass.cmake b/Tests/RunCMake/CTestCommandLine/test1-pass.cmake new file mode 100644 index 0000000000..dda8deae50 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/test1-pass.cmake @@ -0,0 +1,13 @@ +# This is run by test test1 in repeat-until-pass-cmake.cmake with cmake -P. +# It reads the file TEST_OUTPUT_FILE and increments the number +# found in the file by 1. Unless the number is 2, then the +# code sends out a cmake error causing the test to pass only on +# the second time it is run. +message("TEST_OUTPUT_FILE = ${TEST_OUTPUT_FILE}") +file(READ "${TEST_OUTPUT_FILE}" COUNT) +message("COUNT= ${COUNT}") +math(EXPR COUNT "${COUNT} + 1") +file(WRITE "${TEST_OUTPUT_FILE}" "${COUNT}") +if(NOT COUNT EQUAL 2) + message(FATAL_ERROR "this test passes only on the 2nd run") +endif() https://cmake.org/gitweb?p=cmake.git;a=commitdiff;h=0187e522448c8fe92c15d4a983afac36950d1d59 commit 0187e522448c8fe92c15d4a983afac36950d1d59 Author: Brad King <brad.k...@kitware.com> AuthorDate: Tue Oct 29 13:41:52 2019 -0400 Commit: Brad King <brad.k...@kitware.com> CommitDate: Tue Oct 29 13:47:23 2019 -0400 cmCTestRunTest: Use inline member initializers diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx index 7f7f736799..2e7836a10a 100644 --- a/Source/CTest/cmCTestRunTest.cxx +++ b/Source/CTest/cmCTestRunTest.cxx @@ -34,9 +34,6 @@ cmCTestRunTest::cmCTestRunTest(cmCTestMultiProcessHandler& multiHandler) this->TestResult.Status = cmCTestTestHandler::NOT_RUN; this->TestResult.TestCount = 0; this->TestResult.Properties = nullptr; - this->NumberOfRunsLeft = 1; // default to 1 run of the test - this->RunUntilFail = false; // default to run the test once - this->RunAgain = false; // default to not having to run again } void cmCTestRunTest::CheckOutput(std::string const& line) diff --git a/Source/CTest/cmCTestRunTest.h b/Source/CTest/cmCTestRunTest.h index 085a6b8f84..7b202d1e9f 100644 --- a/Source/CTest/cmCTestRunTest.h +++ b/Source/CTest/cmCTestRunTest.h @@ -129,9 +129,9 @@ private: std::vector<std::map< std::string, std::vector<cmCTestMultiProcessHandler::HardwareAllocation>>> AllocatedHardware; - bool RunUntilFail; - int NumberOfRunsLeft; - bool RunAgain; + bool RunUntilFail = false; // default to run the test once + int NumberOfRunsLeft = 1; // default to 1 run of the test + bool RunAgain = false; // default to not having to run again size_t TotalNumberOfTests; }; ----------------------------------------------------------------------- Summary of changes: Help/manual/ctest.1.rst | 13 +++++ Help/release/dev/ctest-repeat-until-pass.rst | 6 +++ Source/CTest/cmCTestMultiProcessHandler.cxx | 4 +- Source/CTest/cmCTestRunTest.cxx | 20 +++++--- Source/CTest/cmCTestRunTest.h | 19 +++++--- Source/cmCTest.cxx | 57 ++++++++++++++++++++-- Source/cmCTest.h | 10 +++- Source/ctest.cxx | 7 ++- Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake | 57 ++++++++++++++++++++-- .../repeat-after-timeout-bad1-result.txt} | 0 .../repeat-after-timeout-bad1-stderr.txt | 1 + .../repeat-after-timeout-bad2-result.txt} | 0 .../repeat-after-timeout-bad2-stderr.txt | 1 + .../repeat-after-timeout-cmake.cmake | 15 ++++++ .../repeat-after-timeout-ctest-stdout.txt | 15 ++++++ ...rr.txt => repeat-after-timeout-good-stderr.txt} | 0 .../repeat-until-fail-and-pass-result.txt} | 0 .../repeat-until-fail-and-pass-stderr.txt | 1 + .../repeat-until-fail-and-timeout-result.txt} | 0 .../repeat-until-fail-and-timeout-stderr.txt | 1 + .../repeat-until-pass-and-fail-result.txt} | 0 .../repeat-until-pass-and-fail-stderr.txt | 1 + .../repeat-until-pass-bad1-result.txt} | 0 .../repeat-until-pass-bad1-stderr.txt | 1 + .../repeat-until-pass-bad2-result.txt} | 0 .../repeat-until-pass-bad2-stderr.txt | 1 + .../CTestCommandLine/repeat-until-pass-cmake.cmake | 15 ++++++ .../repeat-until-pass-ctest-stdout.txt | 15 ++++++ ...tderr.txt => repeat-until-pass-good-stderr.txt} | 0 Tests/RunCMake/CTestCommandLine/test1-pass.cmake | 13 +++++ .../RunCMake/CTestCommandLine/test1-timeout.cmake | 14 ++++++ 31 files changed, 259 insertions(+), 28 deletions(-) create mode 100644 Help/release/dev/ctest-repeat-until-pass.rst copy Tests/RunCMake/{Android/BadSYSROOT-result.txt => CTestCommandLine/repeat-after-timeout-bad1-result.txt} (100%) create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-stderr.txt copy Tests/RunCMake/{Android/BadSYSROOT-result.txt => CTestCommandLine/repeat-after-timeout-bad2-result.txt} (100%) create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-after-timeout-cmake.cmake create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-after-timeout-ctest-stdout.txt copy Tests/RunCMake/CTestCommandLine/{repeat-until-fail-good-stderr.txt => repeat-after-timeout-good-stderr.txt} (100%) copy Tests/RunCMake/{Android/BadSYSROOT-result.txt => CTestCommandLine/repeat-until-fail-and-pass-result.txt} (100%) create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-stderr.txt copy Tests/RunCMake/{Android/BadSYSROOT-result.txt => CTestCommandLine/repeat-until-fail-and-timeout-result.txt} (100%) create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-stderr.txt copy Tests/RunCMake/{Android/BadSYSROOT-result.txt => CTestCommandLine/repeat-until-pass-and-fail-result.txt} (100%) create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-stderr.txt copy Tests/RunCMake/{Android/BadSYSROOT-result.txt => CTestCommandLine/repeat-until-pass-bad1-result.txt} (100%) create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-stderr.txt copy Tests/RunCMake/{Android/BadSYSROOT-result.txt => CTestCommandLine/repeat-until-pass-bad2-result.txt} (100%) create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-pass-cmake.cmake create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-pass-ctest-stdout.txt copy Tests/RunCMake/CTestCommandLine/{repeat-until-fail-good-stderr.txt => repeat-until-pass-good-stderr.txt} (100%) create mode 100644 Tests/RunCMake/CTestCommandLine/test1-pass.cmake create mode 100644 Tests/RunCMake/CTestCommandLine/test1-timeout.cmake hooks/post-receive -- CMake _______________________________________________ Cmake-commits mailing list Cmake-commits@cmake.org https://cmake.org/mailman/listinfo/cmake-commits