Re: [CMake] CTest continuous testing
The devil is always in the little details. That does the trick nicely, thanks!
Tim
- Original Message -
From: "David Cole"
To: gtg0...@mail.gatech.edu
Cc: cmake@cmake.org
Sent: Friday, January 13, 2012 2:37:29 PM
Subject: Re: [CMake] CTest continuous testing
See the cmake_common.cmake script (as seen here on a CMake Continuous
dashboard, and in other places):
http://cdash.org/CDash/viewNotes.php?buildid=1911520
Notice that it does:
ctest_update(RETURN_VALUE count)
and then checks whether count is > 0 or not. It only does stuff if
count > 0 or it's the first time, or it's not a continuous
dashboard...
You need logic like that, and do not execute the configure/build/test
steps when nothing came back from the update...
HTH,
David
On Fri, Jan 13, 2012 at 2:11 PM, Tim Gallagher wrote:
> Hi,
>
> We have a test case suite setup to run in continuous mode and submit to
> CDash. I followed instructions on the wiki
> (http://www.cmake.org/Wiki/CMake_Scripting_Of_CTest#Continuous_Builds_.28new_Style.29)
> and have a script that's below the signature.
>
> I was under the impression that continuous testing would run once when it
> started each run, then sleep and check for changed files and if nothing
> changed, go back to sleep and wait again, then check again, etc.. And if
> files did change and ctest_update() pulled in new code, the tests would run
> again.
>
> However, the behavior I am getting is that the entire test suite is run every
> 5 minutes for the entire continuous run time, even though no files are being
> updated. Some of the tests are failing, if that makes a difference.
>
> Was I mistaken on how it works? Or did I miss a step somewhere? Thanks,
>
> Tim
>
> ---
> while(${CTEST_ELAPSED_TIME} LESS 43200)
> set(START_TIME ${CTEST_ELAPSED_TIME})
>
> ctest_start("Continuous")
> ctest_update()
> # Copy over the files we need
> configure_file(${CTEST_REPO_DIRECTORY}/CTest/CTestCustom.cmake
> ${CTEST_REPO_DIRECTORY}/CTestCustom.cmake @ONLY)
> configure_file(${CTEST_REPO_DIRECTORY}/CTest/CTestTestfile.cmake
> ${CTEST_REPO_DIRECTORY}/CTestTestfile.cmake @ONLY)
> configure_file(${CTEST_REPO_DIRECTORY}/CTest/CTestAvailableTests.cmake
> ${CTEST_REPO_DIRECTORY}/CTestAvailableTests.cmake @ONLY)
> configure_file(${CTEST_REPO_DIRECTORY}/CTest/runPythonTest.sh.in
> ${CTEST_REPO_DIRECTORY}/runPythonTest.sh @ONLY)
>
> include(${CTEST_REPO_DIRECTORY}/CTestAvailableTests.cmake)
>
> foreach(TESTCASE ${LESLIE_AVAILABLE_TESTCASES})
> set(CTEST_CONFIGURE_COMMAND "./setup.py -t ${TESTCASE}
> '{${LESLIE_CONFIGURE_DICT_BASE}}'")
> ctest_configure(BUILD "${CTEST_REPO_DIRECTORY}" APPEND)
> set(CTEST_BUILD_COMMAND "./setup.py -t ${TESTCASE}
> '{${LESLIE_BUILD_DICT_BASE}}'")
> ctest_build(BUILD "${CTEST_REPO_DIRECTORY}" APPEND)
> endforeach()
>
> list(LENGTH LESLIE_CUSTOM_TESTCASE_BASE len1)
> math(EXPR len2 "${len1} - 1")
>
> foreach(value RANGE ${len2})
> list(GET LESLIE_CUSTOM_TESTCASE_BASE ${value} TESTCASE_BASE)
> list(GET LESLIE_CUSTOM_TESTCASE_CONFIG_DICT ${value} TESTCASE_CONFIG)
> list(GET LESLIE_CUSTOM_TESTCASE_BUILD_DICT ${value} TESTCASE_BUILD)
>
> set(CTEST_CONFIGURE_COMMAND "./setup.py -t ${TESTCASE_BASE}
> ${TESTCASE_CONFIG}")
> ctest_configure(BUILD "${CTEST_REPO_DIRECTORY}" APPEND)
> set(CTEST_BUILD_COMMAND "./setup.py -t ${TESTCASE_BASE} ${TESTCASE_BUILD}")
> ctest_build(BUILD "${CTEST_REPO_DIRECTORY}" APPEND)
> endforeach()
>
> ctest_test(BUILD "${CTEST_REPO_DIRECTORY}")
> ctest_submit()
>
> ctest_sleep(${START_TIME} 300 ${CTEST_ELAPSED_TIME})
> endwhile()
>
>
>
> --
>
> Powered by www.kitware.com
>
> Visit other Kitware open-source projects at
> http://www.kitware.com/opensource/opensource.html
>
> Please keep messages on-topic and check the CMake FAQ at:
> http://www.cmake.org/Wiki/CMake_FAQ
>
> Follow this link to subscribe/unsubscribe:
> http://www.cmake.org/mailman/listinfo/cmake
--
Powered by www.kitware.com
Visit other Kitware open-source projects at
http://www.kitware.com/opensource/opensource.html
Please keep messages on-topic and check the CMake FAQ at:
http://www.cmake.org/Wiki/CMake_FAQ
Follow this link to subscribe/unsubscribe:
http://www.cmake.org/mailman/listinfo/cmake
Re: [CMake] CTest continuous testing
See the cmake_common.cmake script (as seen here on a CMake Continuous
dashboard, and in other places):
http://cdash.org/CDash/viewNotes.php?buildid=1911520
Notice that it does:
ctest_update(RETURN_VALUE count)
and then checks whether count is > 0 or not. It only does stuff if
count > 0 or it's the first time, or it's not a continuous
dashboard...
You need logic like that, and do not execute the configure/build/test
steps when nothing came back from the update...
HTH,
David
On Fri, Jan 13, 2012 at 2:11 PM, Tim Gallagher wrote:
> Hi,
>
> We have a test case suite setup to run in continuous mode and submit to
> CDash. I followed instructions on the wiki
> (http://www.cmake.org/Wiki/CMake_Scripting_Of_CTest#Continuous_Builds_.28new_Style.29)
> and have a script that's below the signature.
>
> I was under the impression that continuous testing would run once when it
> started each run, then sleep and check for changed files and if nothing
> changed, go back to sleep and wait again, then check again, etc.. And if
> files did change and ctest_update() pulled in new code, the tests would run
> again.
>
> However, the behavior I am getting is that the entire test suite is run every
> 5 minutes for the entire continuous run time, even though no files are being
> updated. Some of the tests are failing, if that makes a difference.
>
> Was I mistaken on how it works? Or did I miss a step somewhere? Thanks,
>
> Tim
>
> ---
> while(${CTEST_ELAPSED_TIME} LESS 43200)
> set(START_TIME ${CTEST_ELAPSED_TIME})
>
> ctest_start("Continuous")
> ctest_update()
> # Copy over the files we need
> configure_file(${CTEST_REPO_DIRECTORY}/CTest/CTestCustom.cmake
> ${CTEST_REPO_DIRECTORY}/CTestCustom.cmake @ONLY)
> configure_file(${CTEST_REPO_DIRECTORY}/CTest/CTestTestfile.cmake
> ${CTEST_REPO_DIRECTORY}/CTestTestfile.cmake @ONLY)
> configure_file(${CTEST_REPO_DIRECTORY}/CTest/CTestAvailableTests.cmake
> ${CTEST_REPO_DIRECTORY}/CTestAvailableTests.cmake @ONLY)
> configure_file(${CTEST_REPO_DIRECTORY}/CTest/runPythonTest.sh.in
> ${CTEST_REPO_DIRECTORY}/runPythonTest.sh @ONLY)
>
> include(${CTEST_REPO_DIRECTORY}/CTestAvailableTests.cmake)
>
> foreach(TESTCASE ${LESLIE_AVAILABLE_TESTCASES})
> set(CTEST_CONFIGURE_COMMAND "./setup.py -t ${TESTCASE}
> '{${LESLIE_CONFIGURE_DICT_BASE}}'")
> ctest_configure(BUILD "${CTEST_REPO_DIRECTORY}" APPEND)
> set(CTEST_BUILD_COMMAND "./setup.py -t ${TESTCASE}
> '{${LESLIE_BUILD_DICT_BASE}}'")
> ctest_build(BUILD "${CTEST_REPO_DIRECTORY}" APPEND)
> endforeach()
>
> list(LENGTH LESLIE_CUSTOM_TESTCASE_BASE len1)
> math(EXPR len2 "${len1} - 1")
>
> foreach(value RANGE ${len2})
> list(GET LESLIE_CUSTOM_TESTCASE_BASE ${value} TESTCASE_BASE)
> list(GET LESLIE_CUSTOM_TESTCASE_CONFIG_DICT ${value} TESTCASE_CONFIG)
> list(GET LESLIE_CUSTOM_TESTCASE_BUILD_DICT ${value} TESTCASE_BUILD)
>
> set(CTEST_CONFIGURE_COMMAND "./setup.py -t ${TESTCASE_BASE}
> ${TESTCASE_CONFIG}")
> ctest_configure(BUILD "${CTEST_REPO_DIRECTORY}" APPEND)
> set(CTEST_BUILD_COMMAND "./setup.py -t ${TESTCASE_BASE} ${TESTCASE_BUILD}")
> ctest_build(BUILD "${CTEST_REPO_DIRECTORY}" APPEND)
> endforeach()
>
> ctest_test(BUILD "${CTEST_REPO_DIRECTORY}")
> ctest_submit()
>
> ctest_sleep(${START_TIME} 300 ${CTEST_ELAPSED_TIME})
> endwhile()
>
>
>
> --
>
> Powered by www.kitware.com
>
> Visit other Kitware open-source projects at
> http://www.kitware.com/opensource/opensource.html
>
> Please keep messages on-topic and check the CMake FAQ at:
> http://www.cmake.org/Wiki/CMake_FAQ
>
> Follow this link to subscribe/unsubscribe:
> http://www.cmake.org/mailman/listinfo/cmake
--
Powered by www.kitware.com
Visit other Kitware open-source projects at
http://www.kitware.com/opensource/opensource.html
Please keep messages on-topic and check the CMake FAQ at:
http://www.cmake.org/Wiki/CMake_FAQ
Follow this link to subscribe/unsubscribe:
http://www.cmake.org/mailman/listinfo/cmake
[CMake] CTest continuous testing
Hi,
We have a test case suite setup to run in continuous mode and submit to CDash.
I followed instructions on the wiki
(http://www.cmake.org/Wiki/CMake_Scripting_Of_CTest#Continuous_Builds_.28new_Style.29)
and have a script that's below the signature.
I was under the impression that continuous testing would run once when it
started each run, then sleep and check for changed files and if nothing
changed, go back to sleep and wait again, then check again, etc.. And if files
did change and ctest_update() pulled in new code, the tests would run again.
However, the behavior I am getting is that the entire test suite is run every 5
minutes for the entire continuous run time, even though no files are being
updated. Some of the tests are failing, if that makes a difference.
Was I mistaken on how it works? Or did I miss a step somewhere? Thanks,
Tim
---
while(${CTEST_ELAPSED_TIME} LESS 43200)
set(START_TIME ${CTEST_ELAPSED_TIME})
ctest_start("Continuous")
ctest_update()
# Copy over the files we need
configure_file(${CTEST_REPO_DIRECTORY}/CTest/CTestCustom.cmake
${CTEST_REPO_DIRECTORY}/CTestCustom.cmake @ONLY)
configure_file(${CTEST_REPO_DIRECTORY}/CTest/CTestTestfile.cmake
${CTEST_REPO_DIRECTORY}/CTestTestfile.cmake @ONLY)
configure_file(${CTEST_REPO_DIRECTORY}/CTest/CTestAvailableTests.cmake
${CTEST_REPO_DIRECTORY}/CTestAvailableTests.cmake @ONLY)
configure_file(${CTEST_REPO_DIRECTORY}/CTest/runPythonTest.sh.in
${CTEST_REPO_DIRECTORY}/runPythonTest.sh @ONLY)
include(${CTEST_REPO_DIRECTORY}/CTestAvailableTests.cmake)
foreach(TESTCASE ${LESLIE_AVAILABLE_TESTCASES})
set(CTEST_CONFIGURE_COMMAND "./setup.py -t ${TESTCASE}
'{${LESLIE_CONFIGURE_DICT_BASE}}'")
ctest_configure(BUILD "${CTEST_REPO_DIRECTORY}" APPEND)
set(CTEST_BUILD_COMMAND "./setup.py -t ${TESTCASE}
'{${LESLIE_BUILD_DICT_BASE}}'")
ctest_build(BUILD "${CTEST_REPO_DIRECTORY}" APPEND)
endforeach()
list(LENGTH LESLIE_CUSTOM_TESTCASE_BASE len1)
math(EXPR len2 "${len1} - 1")
foreach(value RANGE ${len2})
list(GET LESLIE_CUSTOM_TESTCASE_BASE ${value} TESTCASE_BASE)
list(GET LESLIE_CUSTOM_TESTCASE_CONFIG_DICT ${value} TESTCASE_CONFIG)
list(GET LESLIE_CUSTOM_TESTCASE_BUILD_DICT ${value} TESTCASE_BUILD)
set(CTEST_CONFIGURE_COMMAND "./setup.py -t ${TESTCASE_BASE}
${TESTCASE_CONFIG}")
ctest_configure(BUILD "${CTEST_REPO_DIRECTORY}" APPEND)
set(CTEST_BUILD_COMMAND "./setup.py -t ${TESTCASE_BASE} ${TESTCASE_BUILD}")
ctest_build(BUILD "${CTEST_REPO_DIRECTORY}" APPEND)
endforeach()
ctest_test(BUILD "${CTEST_REPO_DIRECTORY}")
ctest_submit()
ctest_sleep(${START_TIME} 300 ${CTEST_ELAPSED_TIME})
endwhile()
--
Powered by www.kitware.com
Visit other Kitware open-source projects at
http://www.kitware.com/opensource/opensource.html
Please keep messages on-topic and check the CMake FAQ at:
http://www.cmake.org/Wiki/CMake_FAQ
Follow this link to subscribe/unsubscribe:
http://www.cmake.org/mailman/listinfo/cmake
