[
https://issues.apache.org/jira/browse/ARROW-2231?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16381217#comment-16381217
]
ASF GitHub Bot commented on ARROW-2231:
---------------------------------------
cpcloud closed pull request #1679: ARROW-2231: [CI] Use clcache on AppVeyor for
faster builds
URL: https://github.com/apache/arrow/pull/1679
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/appveyor.yml b/appveyor.yml
index 22d6158be..42c0ae34c 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -26,56 +26,69 @@ only_commits:
- cpp/
- python/
+cache:
+ - C:\Users\Appveyor\clcache
+
environment:
matrix:
+ # NOTE: clcache seems to work best with Ninja and worst with msbuild
+ # (as generated by cmake)
- JOB: "Build"
- GENERATOR: Visual Studio 15 2017 Win64
- PYTHON: "3.5"
+ GENERATOR: Ninja
+ PYTHON: "3.6"
ARCH: "64"
CONFIGURATION: "Release"
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
BOOST_ROOT: C:\Libraries\boost_1_64_0
- JOB: "Build_Debug"
- GENERATOR: Visual Studio 14 2015 Win64
- PYTHON: "3.5"
+ GENERATOR: Ninja
+ PYTHON: "3.6"
ARCH: "64"
CONFIGURATION: "Debug"
- JOB: "Build"
- GENERATOR: Visual Studio 14 2015 Win64
- PYTHON: "3.5"
+ GENERATOR: Ninja
+ PYTHON: "3.6"
ARCH: "64"
CONFIGURATION: "Release"
- JOB: "Static_Crt_Build"
- GENERATOR: Visual Studio 14 2015 Win64
- PYTHON: "3.5"
+ GENERATOR: Ninja
+ PYTHON: "3.6"
ARCH: "64"
- JOB: "Toolchain"
GENERATOR: Visual Studio 14 2015 Win64
- PYTHON: "3.5"
+ PYTHON: "3.6"
ARCH: "64"
CONFIGURATION: "Release"
- JOB: "Cmake_Script_Tests"
GENERATOR: NMake Makefiles
- PYTHON: "3.5"
+ PYTHON: "3.6"
ARCH: "64"
CONFIGURATION: "Release"
- JOB: "Build"
GENERATOR: NMake Makefiles
- PYTHON: "3.5"
+ PYTHON: "3.6"
ARCH: "64"
CONFIGURATION: "Release"
MSVC_DEFAULT_OPTIONS: ON
BOOST_ROOT: C:\Libraries\boost_1_63_0
BOOST_LIBRARYDIR: C:\Libraries\boost_1_63_0\lib64-msvc-14.0
+ APPVEYOR_SAVE_CACHE_ON_ERROR: true
+ USE_CLCACHE: true
-init:
- - set MINICONDA=C:\Miniconda35-x64
+install:
+ - set MINICONDA=C:\Miniconda36-x64
- set PATH=%MINICONDA%;%MINICONDA%/Scripts;%MINICONDA%/Library/bin;%PATH%
- - if "%GENERATOR%"=="NMake Makefiles" call "C:\Program Files (x86)\Microsoft
Visual Studio 14.0\VC\vcvarsall.bat" amd64
+ - call ci\appveyor-setup.bat
build_script:
- git config core.symlinks true
- git reset --hard
- if "%JOB%"=="Cmake_Script_Tests" call ci\test-cmake-build-script.bat
- if NOT "%JOB%"=="Cmake_Script_Tests" call ci\msvc-build.bat
+
+# Disable test discovery
+test: off
+
+after_build:
+ - if "%USE_CLCACHE%" == "true" clcache -s
diff --git a/ci/appveyor-setup.bat b/ci/appveyor-setup.bat
new file mode 100644
index 000000000..34f8f1d33
--- /dev/null
+++ b/ci/appveyor-setup.bat
@@ -0,0 +1,56 @@
+@rem Licensed to the Apache Software Foundation (ASF) under one
+@rem or more contributor license agreements. See the NOTICE file
+@rem distributed with this work for additional information
+@rem regarding copyright ownership. The ASF licenses this file
+@rem to you under the Apache License, Version 2.0 (the
+@rem "License"); you may not use this file except in compliance
+@rem with the License. You may obtain a copy of the License at
+@rem
+@rem http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing,
+@rem software distributed under the License is distributed on an
+@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@rem KIND, either express or implied. See the License for the
+@rem specific language governing permissions and limitations
+@rem under the License.
+
+@echo on
+
+conda update -y -q conda
+conda config --set auto_update_conda false
+conda info -a
+
+conda config --set show_channel_urls True
+
+@rem Help with SSL timeouts to S3
+conda config --set remote_connect_timeout_secs 12
+
+conda config --add channels https://repo.continuum.io/pkgs/free
+conda config --add channels conda-forge
+conda info -a
+
+if "%GENERATOR%"=="NMake Makefiles" set need_vcvarsall=1
+if "%GENERATOR%"=="Ninja" set need_vcvarsall=1
+
+if defined need_vcvarsall (
+ @rem Select desired compiler version
+ if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2017" (
+ call "C:\Program Files (x86)\Microsoft Visual
Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat" amd64
+ ) else (
+ call "C:\Program Files (x86)\Microsoft Visual Studio
14.0\VC\vcvarsall.bat" amd64
+ )
+)
+
+if "%GENERATOR%"=="Ninja" conda install -y -q -c conda-forge ninja
+
+if "%USE_CLCACHE%" == "true" (
+ @rem Use clcache for faster builds
+ pip install -q git+https://github.com/frerich/clcache.git
+ clcache -s
+ set CC=clcache
+ set CXX=clcache
+ set CLCACHE_SERVER=1
+ set CLCACHE_HARDLINK=1
+ powershell.exe -Command "Start-Process clcache-server"
+)
diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index 069ddf0fc..b6f672da9 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -29,6 +29,7 @@ if "%JOB%" == "Static_Crt_Build" (
.. || exit /B
cmake --build . --config Debug || exit /B
+ ctest -VV || exit /B
popd
mkdir cpp\build-release
@@ -60,42 +61,27 @@ if "%JOB%" == "Build_Debug" (
.. || exit /B
cmake --build . --config Debug || exit /B
+ ctest -VV || exit /B
popd
@rem Finish Debug build successfully
exit /B 0
)
-conda update --yes --quiet conda
-conda config --set auto_update_conda false
-conda info -a
-
-conda config --set show_channel_urls True
-
-# Help with SSL timeouts to S3
-conda config --set remote_connect_timeout_secs 12
-
-conda config --add channels https://repo.continuum.io/pkgs/free
-conda config --add channels conda-forge
-conda info -a
-
conda create -n arrow -q -y python=%PYTHON% ^
six pytest setuptools numpy pandas cython ^
thrift-cpp=0.11.0
-if "%JOB%" == "Toolchain" (
+call activate arrow
- conda install -n arrow -q -y -c conda-forge ^
+if "%JOB%" == "Toolchain" (
+ @rem Install pre-built "toolchain" packages for faster builds
+ conda install -q -y -c conda-forge ^
flatbuffers rapidjson ^
cmake ^
git ^
boost-cpp ^
snappy zlib brotli gflags lz4-c zstd
-)
-
-call activate arrow
-
-if "%JOB%" == "Toolchain" (
set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
)
@@ -113,12 +99,15 @@ cmake -G "%GENERATOR%" ^
-DARROW_CXXFLAGS="/WX /MP" ^
-DARROW_PYTHON=ON ^
.. || exit /B
-cmake --build . --target INSTALL --config %CONFIGURATION% || exit /B
+cmake --build . --target install --config %CONFIGURATION% || exit /B
@rem Needed so python-test.exe works
+set OLD_PYTHONPATH=%PYTHONPATH%
set
PYTHONPATH=%CONDA_PREFIX%\Lib;%CONDA_PREFIX%\Lib\site-packages;%CONDA_PREFIX%\python35.zip;%CONDA_PREFIX%\DLLs;%CONDA_PREFIX%;%PYTHONPATH%
ctest -VV || exit /B
+
+set PYTHONPATH=%OLD_PYTHONPATH%
popd
@rem Build parquet-cpp
@@ -134,7 +123,7 @@ cmake -G "%GENERATOR%" ^
-DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DPARQUET_BOOST_USE_SHARED=OFF ^
-DPARQUET_BUILD_TESTS=off .. || exit /B
-cmake --build . --target INSTALL --config %CONFIGURATION% || exit /B
+cmake --build . --target install --config %CONFIGURATION% || exit /B
popd
@rem Build and install pyarrow
diff --git a/ci/test-cmake-build-script.bat b/ci/test-cmake-build-script.bat
index 54d892fd4..94857e823 100644
--- a/ci/test-cmake-build-script.bat
+++ b/ci/test-cmake-build-script.bat
@@ -17,13 +17,10 @@
@echo on
-conda update --yes --quiet conda
-
@rem Validate cmake script behaviour on missed lib in toolchain
set CONDA_ENV=arrow-cmake-tests-libs
conda create -n %CONDA_ENV% -q -y
-conda install -n %CONDA_ENV% -q -y -c conda-forge ^
-cmake git boost-cpp
+conda install -n %CONDA_ENV% -q -y -c conda-forge boost-cpp
call activate %CONDA_ENV%
set BUILD_DIR=cpp\build-cmake-test
@@ -125,7 +122,7 @@ pushd %BUILD_DIR%
set CONDA_ENV=arrow-cmake-tests-toolchain
conda create -n %CONDA_ENV% -q -y
conda install -n %CONDA_ENV% -q -y -c conda-forge ^
- flatbuffers rapidjson cmake git boost-cpp ^
+ flatbuffers rapidjson boost-cpp ^
thrift-cpp snappy zlib brotli gflags lz4-c zstd
call activate %CONDA_ENV%
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc
b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index d877e9922..c1ff8a448 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -461,6 +461,8 @@ TEST_F(RecursionLimits, ReadLimit) {
ASSERT_RAISES(Invalid, ReadRecordBatch(*message->metadata(), schema,
&reader, &result));
}
+// Test fails with a structured exception on Windows + Debug
+#if !defined(_WIN32) || defined(NDEBUG)
TEST_F(RecursionLimits, StressLimit) {
auto CheckDepth = [this](int recursion_depth, bool* it_works) {
int32_t metadata_length = -1;
@@ -487,6 +489,7 @@ TEST_F(RecursionLimits, StressLimit) {
CheckDepth(500, &it_works);
ASSERT_TRUE(it_works);
}
+#endif // !defined(_WIN32) || defined(NDEBUG)
class TestFileFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
public:
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> [CI] Use clcache on AppVeyor
> ----------------------------
>
> Key: ARROW-2231
> URL: https://issues.apache.org/jira/browse/ARROW-2231
> Project: Apache Arrow
> Issue Type: Improvement
> Components: Continuous Integration
> Affects Versions: 0.8.0
> Environment: Windows
> Reporter: Antoine Pitrou
> Assignee: Antoine Pitrou
> Priority: Major
> Labels: CI, pull-request-available, windows
> Fix For: 0.9.0
>
>
> There's a ccache equivalent for Windows named clcache that could be used to
> speed up AppVeyor builds: https://github.com/frerich/clcache/
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
