Repository: arrow Updated Branches: refs/heads/master cab07c2bd -> bc16e0e04
ARROW-1196: [C++] Release, Debug, Toolchain, NMake Generator Appveyor⦠⦠jobs; Windows.md updated. Author: Max Risuhin <risuhin....@gmail.com> Closes #823 from MaxRis/ARROW-1196 and squashes the following commits: 1ca1155 [Max Risuhin] ARROW-1196: [C++] Release, Debug, Toolchain, NMake Generator Appveyor jobs; Windows.md updated. Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/bc16e0e0 Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/bc16e0e0 Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/bc16e0e0 Branch: refs/heads/master Commit: bc16e0e04a3d4ea9c96579ac59cee4b16e771850 Parents: cab07c2 Author: Max Risuhin <risuhin....@gmail.com> Authored: Mon Jul 10 19:32:46 2017 +0200 Committer: Uwe L. Korn <uw...@xhochy.com> Committed: Mon Jul 10 19:32:46 2017 +0200 ---------------------------------------------------------------------- appveyor.yml | 15 +++ ci/msvc-build.bat | 31 ++++- cpp/CMakeLists.txt | 4 + cpp/apidoc/Windows.md | 146 ++++++++++++++++++++--- cpp/cmake_modules/ThirdpartyToolchain.cmake | 7 +- 5 files changed, 181 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/bc16e0e0/appveyor.yml ---------------------------------------------------------------------- diff --git a/appveyor.yml b/appveyor.yml index 802d43b..f3ce5a5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -20,9 +20,23 @@ os: Visual Studio 2015 environment: matrix: + - GENERATOR: NMake Makefiles + PYTHON: "3.5" + ARCH: "64" + CONFIGURATION: "Release" + - GENERATOR: Visual Studio 14 2015 Win64 + PYTHON: "3.5" + ARCH: "64" + CONFIGURATION: "Debug" - GENERATOR: Visual Studio 14 2015 Win64 PYTHON: "3.5" ARCH: "64" + CONFIGURATION: "Release" + - GENERATOR: Visual Studio 14 2015 Win64 + PYTHON: "3.5" + ARCH: "64" + CONFIGURATION: "Toolchain" + MSVC_DEFAULT_OPTIONS: ON BOOST_ROOT: C:\Libraries\boost_1_63_0 BOOST_LIBRARYDIR: C:\Libraries\boost_1_63_0\lib64-msvc-14.0 @@ -30,6 +44,7 @@ environment: init: - set MINICONDA=C:\Miniconda35-x64 - set PATH=%MINICONDA%;%MINICONDA%/Scripts;%MINICONDA%/Library/bin;%PATH% + - if "%GENERATOR%"=="NMake Makefiles" call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86_amd64 build_script: - git config core.symlinks true http://git-wip-us.apache.org/repos/asf/arrow/blob/bc16e0e0/ci/msvc-build.bat ---------------------------------------------------------------------- diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat index 366fc28..c84d12d 100644 --- a/ci/msvc-build.bat +++ b/ci/msvc-build.bat @@ -17,18 +17,41 @@ @echo on +if "%CONFIGURATION%" == "Debug" ( + mkdir cpp\build-debug + pushd cpp\build-debug + + cmake -G "%GENERATOR%" ^ + -DARROW_BOOST_USE_SHARED=OFF ^ + -DCMAKE_BUILD_TYPE=Debug ^ + -DARROW_CXXFLAGS="/MP" ^ + .. || exit /B + + cmake --build . --config Debug || exit /B + popd + + @rem Finish Debug build successfully + exit /B 0 +) + conda update --yes --quiet conda conda create -n arrow -q -y python=%PYTHON% ^ six pytest setuptools numpy pandas cython -conda install -n arrow -q -y -c conda-forge ^ - flatbuffers rapidjson ^ - cmake git boost-cpp thrift-cpp snappy zlib brotli gflags + +if "%CONFIGURATION%" == "Toolchain" ( + conda install -n arrow -q -y -c conda-forge ^ + flatbuffers rapidjson cmake git boost-cpp ^ + thrift-cpp snappy zlib brotli gflags lz4-c zstd +) call activate arrow +if "%CONFIGURATION%" == "Toolchain" ( + set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library +) + set ARROW_HOME=%CONDA_PREFIX%\Library -set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library @rem Build and test Arrow C++ libraries http://git-wip-us.apache.org/repos/asf/arrow/blob/bc16e0e0/cpp/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 6d01fd9..002a07e 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -152,6 +152,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") "Snappy static lib suffix used on Windows with MSVC (default is empty string)") set(ZLIB_MSVC_STATIC_LIB_SUFFIX "libstatic" CACHE STRING "Zlib static lib suffix used on Windows with MSVC (default libstatic)") + set(LZ4_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING + "Lz4 static lib suffix used on Windows with MSVC (default _static)") + set(ZSTD_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING + "ZStd static lib suffix used on Windows with MSVC (default _static)") endif() endif() http://git-wip-us.apache.org/repos/asf/arrow/blob/bc16e0e0/cpp/apidoc/Windows.md ---------------------------------------------------------------------- diff --git a/cpp/apidoc/Windows.md b/cpp/apidoc/Windows.md index b9a6143..6bfb951 100644 --- a/cpp/apidoc/Windows.md +++ b/cpp/apidoc/Windows.md @@ -21,24 +21,13 @@ investing time in maintaining the thirdparty build dependencies for Arrow and related C++ libraries using the conda package manager. Others are free to add other development instructions for Windows here. -### Visual Studio - -Microsoft provides the free Visual Studio 2017 Community edition. When doing -development, you must launch the developer command prompt using - -``` -"C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64 -``` - -It's easiest to configure a console emulator like [cmder][3] to automatically -launch this when starting a new development console. - ### conda and package toolchain [Miniconda][1] is a minimal Python distribution including the conda package manager. To get started, download and install a 64-bit distribution. -We recommend using packages from [conda-forge][2] +We recommend using packages from [conda-forge][2]. +Launch cmd.exe and run following commands: ```shell conda config --add channels conda-forge @@ -47,9 +36,92 @@ conda config --add channels conda-forge Now, you can bootstrap a build environment ```shell -conda create -n arrow-dev cmake git boost +conda create -n arrow-dev cmake git boost-cpp flatbuffers rapidjson cmake thrift-cpp snappy zlib brotli gflags lz4-c zstd ``` +Activate just created conda environment with pre-installed packages from +previous step: + +```shell +activate arrow-dev +``` + +We are using [cmake][4] tool to support Windows builds. +To allow cmake to pick up 3rd party dependencies, you should set +`ARROW_BUILD_TOOLCHAIN` environment variable to contain `Library` folder +path of new created on previous step `arrow-dev` conda environment. +For instance, if `Miniconda` was installed to default destination, `Library` +folder path for `arrow-dev` conda environment will be as following: + +```shell +C:\Users\YOUR_USER_NAME\Miniconda3\envs\arrow-dev\Library +``` + +To set `ARROW_BUILD_TOOLCHAIN` environment variable visible only for current terminal session you can run following: +```shell +set ARROW_BUILD_TOOLCHAIN=C:\Users\YOUR_USER_NAME\Miniconda3\envs\arrow-dev\Library +``` + +To validate value of `ARROW_BUILD_TOOLCHAIN` environment variable you can run following terminal command: +```shell +echo %ARROW_BUILD_TOOLCHAIN% +``` + +As alternative to `ARROW_BUILD_TOOLCHAIN`, it's possible to configure path +to each 3rd party dependency separately by setting appropriate environment +variable: + +`FLATBUFFERS_HOME` variable with path to `flatbuffers` installation +`RAPIDJSON_HOME` variable with path to `rapidjson` installation +`GFLAGS_HOME` variable with path to `gflags` installation +`SNAPPY_HOME` variable with path to `snappy` installation +`ZLIB_HOME` variable with path to `zlib` installation +`BROTLI_HOME` variable with path to `brotli` installation +`LZ4_HOME` variable with path to `lz4` installation +`ZSTD_HOME` variable with path to `zstd` installation + +### Customize static libraries names lookup of 3rd party dependencies + +If you decided to use pre-built 3rd party dependencies libs, it's possible to +configure Arrow's cmake build script to search for customized names of 3rd +party static libs. + +`zlib`. Pass `-DARROW_ZLIB_VENDORED=OFF` to enable lookup of custom zlib +build. Set `ZLIB_HOME` environment variable. Pass +`-DZLIB_MSVC_STATIC_LIB_SUFFIX=%ZLIB_SUFFIX%` to link with z%ZLIB_SUFFIX%.lib + +`brotli`. Set `BROTLY_HOME` environment variable. Pass +`-DBROTLI_MSVC_STATIC_LIB_SUFFIX=%BROTLI_SUFFIX%` to link with +brotli*%BROTLI_SUFFIX%.lib. + +`snappy`. Set `SNAPPY_HOME` environment variable. Pass +`-DSNAPPY_MSVC_STATIC_LIB_SUFFIX=%SNAPPY_SUFFIX%` to link with +snappy%SNAPPY_SUFFIX%.lib. + +`lz4`. Set `LZ4_HOME` environment variable. Pass +`-LZ4_MSVC_STATIC_LIB_SUFFIX=%LZ4_SUFFIX%` to link with +lz4%LZ4_SUFFIX%.lib. + +`zstd`. Set `ZSTD_HOME` environment variable. Pass +`-ZSTD_MSVC_STATIC_LIB_SUFFIX=%ZSTD_SUFFIX%` to link with +zstd%ZSTD_SUFFIX%.lib. + +### Visual Studio + +Microsoft provides the free Visual Studio Community edition. When doing +development, you must launch the developer command prompt using + +#### Visual Studio 2015 + +```"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64``` + +#### Visual Studio 2017 + +```"C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64``` + +It's easiest to configure a console emulator like [cmder][3] to automatically +launch this when starting a new development console. + ## Building with NMake Activate your conda build environment: @@ -58,7 +130,8 @@ Activate your conda build environment: activate arrow-dev ``` -Now, do an out of source build using `nmake`: +Change working directory in cmd.exe to the root directory of Arrow and +do an out of source build using `nmake`: ``` cd cpp @@ -72,14 +145,53 @@ When using conda, only release builds are currently supported. ## Build using Visual Studio (MSVC) Solution Files -To build on the command line by instead generating a MSVC solution, instead -run: +Activate your conda build environment: ``` +activate arrow-dev +``` + +Change working directory in cmd.exe to the root directory of Arrow and +do an out of source build by generating a MSVC solution: + +``` +cd cpp +mkdir build +cd build cmake -G "Visual Studio 14 2015 Win64" -DCMAKE_BUILD_TYPE=Release .. cmake --build . --config Release ``` +## Debug build + +To build Debug version of Arrow you should have pre-insalled Debug version of +boost libs. + +It's recommended to configure cmake build with following variables for Debug build: + +`-DARROW_BOOST_USE_SHARED=OFF` - enables static linking with boost debug libs and +simplifies run-time loading of 3rd parties. (Recommended) + +`-DBOOST_ROOT` - sets the root directory of boost libs. (Optional) + +`-DBOOST_LIBRARYDIR` - sets the directory with boost lib files. (Optional) + +Command line to build Arrow in Debug might look as following: + +``` +cd cpp +mkdir build +cd build +cmake -G "Visual Studio 14 2015 Win64" -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=OFF -DCMAKE_BUILD_TYPE=Debug -DBOOST_ROOT=C:/local/boost_1_63_0 -DBOOST_LIBRARYDIR=C:/local/boost_1_63_0/lib64-msvc-14.0 .. +cmake --build . --config Debug +``` + +To get the latest build instructions, you can reference [msvc-build.bat][5], which is used by automated Appveyor builds. + + + [1]: https://conda.io/miniconda.html [2]: https://conda-forge.github.io/ [3]: http://cmder.net/ +[4]: https://cmake.org/ +[5]: https://github.com/apache/arrow/blob/master/ci/msvc-build.bat http://git-wip-us.apache.org/repos/asf/arrow/blob/bc16e0e0/cpp/cmake_modules/ThirdpartyToolchain.cmake ---------------------------------------------------------------------- diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 3eef2f7..74939ac 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -346,10 +346,15 @@ if (ARROW_IPC) ## Flatbuffers if("${FLATBUFFERS_HOME}" STREQUAL "") set(FLATBUFFERS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_ep-prefix/src/flatbuffers_ep-install") + if (MSVC) + set(FLATBUFFERS_CMAKE_CXX_FLAGS /EHsc) + else() + set(FLATBUFFERS_CMAKE_CXX_FLAGS -fPIC) + endif() ExternalProject_Add(flatbuffers_ep URL "https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz" CMAKE_ARGS - "-DCMAKE_CXX_FLAGS=-fPIC" + "-DCMAKE_CXX_FLAGS=${FLATBUFFERS_CMAKE_CXX_FLAGS}" "-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_PREFIX}" "-DFLATBUFFERS_BUILD_TESTS=OFF")