[cmake-developers] [CMake 0015749]: Make the URL option of ExternalProject_Add to accept lists
The following issue has been SUBMITTED. == http://www.cmake.org/Bug/view.php?id=15749 == Reported By:Ilya Assigned To: == Project:CMake Issue ID: 15749 Category: CMake Reproducibility:N/A Severity: feature Priority: normal Status: new == Date Submitted: 2015-09-16 22:51 EDT Last Modified: 2015-09-16 22:51 EDT == Summary:Make the URL option of ExternalProject_Add to accept lists Description: It would be handy if one could specify a bunch of mirrors for a specific dependency. == Issue History Date ModifiedUsername FieldChange == 2015-09-16 22:51 Ilya New Issue == -- Powered by www.kitware.com Please keep messages on-topic and check the CMake FAQ at: http://www.cmake.org/Wiki/CMake_FAQ Kitware offers various services to support the CMake community. For more information on each offering, please visit: CMake Support: http://cmake.org/cmake/help/support.html CMake Consulting: http://cmake.org/cmake/help/consulting.html CMake Training Courses: http://cmake.org/cmake/help/training.html Visit other Kitware open-source projects at http://www.kitware.com/opensource/opensource.html Follow this link to subscribe/unsubscribe: http://public.kitware.com/mailman/listinfo/cmake-developers
[cmake-developers] [CMake 0015749]: Make the URL option of ExternalProject_Add to accept lists
Hello there, I try to continue the work from André. With the attached patch it is possible to use a list of URLs but it is still limited to a single path. I think for a mix of pathes/URLs we need to move a lot of code from ExternalProject.cmake into ExternalProject-download.cmake. Maybe it is good to realize this in a new ticket/commit? Best regards Lars Schmertmann From 036b3cb1a459befe7c0917f98c53ec0f176d2a8e Mon Sep 17 00:00:00 2001 From: Lars Schmertmann Date: Tue, 12 Jul 2016 16:53:36 +0200 Subject: [PATCH] Add support for multiple/alternative URLs Now it is possible to pass multiple URLs as a list that will be tried in and foreach(). So it will try next URL if the previous failed. --- Modules/ExternalProject-download.cmake.in | 84 +-- Modules/ExternalProject.cmake | 17 ++- 2 files changed, 61 insertions(+), 40 deletions(-) diff --git a/Modules/ExternalProject-download.cmake.in b/Modules/ExternalProject-download.cmake.in index 5b73cd8..91d74e0 100644 --- a/Modules/ExternalProject-download.cmake.in +++ b/Modules/ExternalProject-download.cmake.in @@ -114,48 +114,56 @@ endif() set(retry_number 5) +message(STATUS "Downloading... + dst='@LOCAL@' + timeout='@TIMEOUT_MSG@'" +) + foreach(i RANGE ${retry_number}) sleep_before_download(${i}) - message(STATUS "downloading... - src='@REMOTE@' - dst='@LOCAL@' - timeout='@TIMEOUT_MSG@'") - - @TLS_VERIFY_CODE@ - @TLS_CAINFO_CODE@ - - file( - DOWNLOAD - "@REMOTE@" "@LOCAL@" - @SHOW_PROGRESS@ - @TIMEOUT_ARGS@ - STATUS status - LOG log - ) - - list(GET status 0 status_code) - list(GET status 1 status_string) - - if(status_code EQUAL 0) -check_file_hash(has_hash hash_is_good) -if(has_hash AND NOT hash_is_good) - message(STATUS "Hash mismatch, removing...") - file(REMOVE "@LOCAL@") + foreach(url @REMOTE@) +message(STATUS "Using src='${url}'") + +@TLS_VERIFY_CODE@ +@TLS_CAINFO_CODE@ + +file( +DOWNLOAD +"${url}" "@LOCAL@" +@SHOW_PROGRESS@ +@TIMEOUT_ARGS@ +STATUS status +LOG log +) + +list(GET status 0 status_code) +list(GET status 1 status_string) + +if(status_code EQUAL 0) + check_file_hash(has_hash hash_is_good) + if(has_hash AND NOT hash_is_good) +message(STATUS "Hash mismatch, removing...") +file(REMOVE "@LOCAL@") + else() +message(STATUS "Downloading... done") +return() + endif() else() - message(STATUS "Downloading... done") - return() + string(APPEND logFailedURLs "error: downloading '${url}' failed + status_code: ${status_code} + status_string: ${status_string} + log: + --- LOG BEGIN --- + ${log} + --- LOG END --- + " + ) endif() - else() -message("error: downloading '@REMOTE@' failed - status_code: ${status_code} - status_string: ${status_string} - log: - --- LOG BEGIN --- - ${log} - --- LOG END ---" -) - endif() + endforeach() endforeach() -message(FATAL_ERROR "Downloading failed") +message(FATAL_ERROR "Each download failed! + ${logFailedURLs} + " +) diff --git a/Modules/ExternalProject.cmake b/Modules/ExternalProject.cmake index 3686fb6..749d5ea 100644 --- a/Modules/ExternalProject.cmake +++ b/Modules/ExternalProject.cmake @@ -69,8 +69,8 @@ Create custom targets to build projects in external trees URL of mercurial repo ``HG_TAG `` Mercurial branch name, commit id or tag - ``URL /.../src.tgz`` -Full path or URL of source + ``URL /.../src.tgz `` +Full path or URL(s) of source. Multiple URLs are allowed as mirrors. ``URL_HASH ALGO=value`` Hash of file at URL ``URL_MD5 md5`` @@ -1861,6 +1861,19 @@ function(_ep_add_download_command name) @ONLY ) list(APPEND depends ${stamp_dir}/${name}-urlinfo.txt) + +list(LENGTH url url_list_length) +if(NOT "${url_list_length}" STREQUAL "1") + foreach(entry ${url}) +if(NOT "${entry}" MATCHES "^[a-z]+://") + message(FATAL_ERROR "At least one entry of URL is a path (invalid in a list)") +endif() + endforeach() + if("x${fname}" STREQUAL "x") +list(GET url 0 fname) + endif() +endif() + if(IS_DIRECTORY "${url}") get_filename_component(abs_dir "${url}" ABSOLUTE) set(comment "Performing download step (DIR copy) for '${name}'") -- 2.9.0 -- Powered by www.kitware.com Please keep messages on-topic and check the CMake FAQ at: http://www.cmake.org/Wiki/CMake_FAQ Kitware offers various services to support the CMake community. For more information on each offering, please visit: CMake Support: http://cmake.org/cmake/help/support.html CMake Consulting: http://cmake.org/cmake/help/consulting.html CMake Training Courses: http://cmake.org/cmake/help/training.html Visit other Kitware open-source projects at http://www.kitware.com/opensourc
Re: [cmake-developers] [CMake 0015749]: Make the URL option of ExternalProject_Add to accept lists
Looks like you could also use a list of paths if you express them in "file://..." form. Local files can also be expressed as URLs. David C. On Wed, Jul 13, 2016 at 4:18 AM, Schmertmann, Lars wrote: > Hello there, > > I try to continue the work from André. > With the attached patch it is possible to use a list of URLs > but it is still limited to a single path. > > I think for a mix of pathes/URLs we need to move a lot of code > from ExternalProject.cmake into ExternalProject-download.cmake. > Maybe it is good to realize this in a new ticket/commit? > > Best regards > Lars Schmertmann > > > -- > > Powered by www.kitware.com > > Please keep messages on-topic and check the CMake FAQ at: > http://www.cmake.org/Wiki/CMake_FAQ > > Kitware offers various services to support the CMake community. For more > information on each offering, please visit: > > CMake Support: http://cmake.org/cmake/help/support.html > CMake Consulting: http://cmake.org/cmake/help/consulting.html > CMake Training Courses: http://cmake.org/cmake/help/training.html > > Visit other Kitware open-source projects at > http://www.kitware.com/opensource/opensource.html > > Follow this link to subscribe/unsubscribe: > http://public.kitware.com/mailman/listinfo/cmake-developers -- Powered by www.kitware.com Please keep messages on-topic and check the CMake FAQ at: http://www.cmake.org/Wiki/CMake_FAQ Kitware offers various services to support the CMake community. For more information on each offering, please visit: CMake Support: http://cmake.org/cmake/help/support.html CMake Consulting: http://cmake.org/cmake/help/consulting.html CMake Training Courses: http://cmake.org/cmake/help/training.html Visit other Kitware open-source projects at http://www.kitware.com/opensource/opensource.html Follow this link to subscribe/unsubscribe: http://public.kitware.com/mailman/listinfo/cmake-developers
Re: [cmake-developers] [CMake 0015749]: Make the URL option of ExternalProject_Add to accept lists
Thanks for the hint. I removed the ^ to replace all occurrences and not only the first one. ExternalProject.cmake, Line 1670: string(REGEX REPLACE "^file://" "" url "${url}") Lars S. Am 13.07.2016 um 11:14 schrieb David Cole: > Looks like you could also use a list of paths if you express them in > "file://..." form. Local files can also be expressed as URLs. > > > David C. > > > > On Wed, Jul 13, 2016 at 4:18 AM, Schmertmann, Lars > wrote: >> Hello there, >> >> I try to continue the work from André. >> With the attached patch it is possible to use a list of URLs >> but it is still limited to a single path. >> >> I think for a mix of pathes/URLs we need to move a lot of code >> from ExternalProject.cmake into ExternalProject-download.cmake. >> Maybe it is good to realize this in a new ticket/commit? >> >> Best regards >> Lars Schmertmann >> >> >> -- >> >> Powered by www.kitware.com >> >> Please keep messages on-topic and check the CMake FAQ at: >> http://www.cmake.org/Wiki/CMake_FAQ >> >> Kitware offers various services to support the CMake community. For more >> information on each offering, please visit: >> >> CMake Support: http://cmake.org/cmake/help/support.html >> CMake Consulting: http://cmake.org/cmake/help/consulting.html >> CMake Training Courses: http://cmake.org/cmake/help/training.html >> >> Visit other Kitware open-source projects at >> http://www.kitware.com/opensource/opensource.html >> >> Follow this link to subscribe/unsubscribe: >> http://public.kitware.com/mailman/listinfo/cmake-developers From 071f1e8af68ffac903c3fdec6bae5ae5a5f3c0c8 Mon Sep 17 00:00:00 2001 From: Lars Schmertmann Date: Tue, 12 Jul 2016 16:53:36 +0200 Subject: [PATCH] Add support for multiple/alternative URLs Now it is possible to pass multiple URLs as a list that will be tried in and foreach(). So it will try next URL if the previous failed. --- Modules/ExternalProject-download.cmake.in | 84 +-- Modules/ExternalProject.cmake | 19 +-- 2 files changed, 62 insertions(+), 41 deletions(-) diff --git a/Modules/ExternalProject-download.cmake.in b/Modules/ExternalProject-download.cmake.in index 5b73cd8..91d74e0 100644 --- a/Modules/ExternalProject-download.cmake.in +++ b/Modules/ExternalProject-download.cmake.in @@ -114,48 +114,56 @@ endif() set(retry_number 5) +message(STATUS "Downloading... + dst='@LOCAL@' + timeout='@TIMEOUT_MSG@'" +) + foreach(i RANGE ${retry_number}) sleep_before_download(${i}) - message(STATUS "downloading... - src='@REMOTE@' - dst='@LOCAL@' - timeout='@TIMEOUT_MSG@'") - - @TLS_VERIFY_CODE@ - @TLS_CAINFO_CODE@ - - file( - DOWNLOAD - "@REMOTE@" "@LOCAL@" - @SHOW_PROGRESS@ - @TIMEOUT_ARGS@ - STATUS status - LOG log - ) - - list(GET status 0 status_code) - list(GET status 1 status_string) - - if(status_code EQUAL 0) -check_file_hash(has_hash hash_is_good) -if(has_hash AND NOT hash_is_good) - message(STATUS "Hash mismatch, removing...") - file(REMOVE "@LOCAL@") + foreach(url @REMOTE@) +message(STATUS "Using src='${url}'") + +@TLS_VERIFY_CODE@ +@TLS_CAINFO_CODE@ + +file( +DOWNLOAD +"${url}" "@LOCAL@" +@SHOW_PROGRESS@ +@TIMEOUT_ARGS@ +STATUS status +LOG log +) + +list(GET status 0 status_code) +list(GET status 1 status_string) + +if(status_code EQUAL 0) + check_file_hash(has_hash hash_is_good) + if(has_hash AND NOT hash_is_good) +message(STATUS "Hash mismatch, removing...") +file(REMOVE "@LOCAL@") + else() +message(STATUS "Downloading... done") +return() + endif() else() - message(STATUS "Downloading... done") - return() + string(APPEND logFailedURLs "error: downloading '${url}' failed + status_code: ${status_code} + status_string: ${status_string} + log: + --- LOG BEGIN --- + ${log} + --- LOG END --- + " + ) endif() - else() -message("error: downloading '@REMOTE@' failed - status_code: ${status_code} - status_string: ${status_string} - log: - --- LOG BEGIN --- - ${log} - --- LOG END ---" -) - endif() + endforeach() endforeach() -message(FATAL_ERROR "Downloading failed") +message(FATAL_ERROR "Each download failed! + ${logFailedURLs} + " +) diff --git a/Modules/ExternalProject.cmake b/Modules/ExternalProject.cmake index 3686fb6..efbb8c1 100644 --- a/Modules/ExternalProject.cmake +++ b/Modules/ExternalProject.cmake @@ -69,8 +69,8 @@ Create custom targets to build projects in external trees URL of mercurial repo ``HG_TAG `` Mercurial branch name, commit id or tag - ``URL /.../src.tgz`` -Full path or URL of source + ``URL /.../src.tgz `` +Full path or URL(s) of source. Multiple URLs are allowed as mirrors. ``URL_HASH ALGO=value`` Hash of file at URL ``URL_MD5 md5`` @@ -1667,7 +1667,7 @@ function(_ep_ad
Re: [cmake-developers] [CMake 0015749]: Make the URL option of ExternalProject_Add to accept lists
On 07/13/2016 05:35 AM, Schmertmann, Lars wrote: > Thanks for the hint. I've merged to `next` for testing: ExternalProject: Add support for multiple alternative URLs https://cmake.org/gitweb?p=cmake.git;a=commitdiff;h=2eec433f >>> I think for a mix of pathes/URLs we need to move a lot of code >>> from ExternalProject.cmake into ExternalProject-download.cmake. >>> Maybe it is good to realize this in a new ticket/commit? Yes. If you're actively working on it then I don't think it needs a separate issue and can just be added as follow-up work. Thanks, -Brad -- Powered by www.kitware.com Please keep messages on-topic and check the CMake FAQ at: http://www.cmake.org/Wiki/CMake_FAQ Kitware offers various services to support the CMake community. For more information on each offering, please visit: CMake Support: http://cmake.org/cmake/help/support.html CMake Consulting: http://cmake.org/cmake/help/consulting.html CMake Training Courses: http://cmake.org/cmake/help/training.html Visit other Kitware open-source projects at http://www.kitware.com/opensource/opensource.html Follow this link to subscribe/unsubscribe: http://public.kitware.com/mailman/listinfo/cmake-developers