This is an automated email from the ASF dual-hosted git repository. tarmstrong pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push: new d0c1030 IMPALA-8147: part 1/2: make make_*.sh redundant. d0c1030 is described below commit d0c103095019f1b80dec63193847eb516d7f8dd7 Author: Tim Armstrong <tarmstr...@cloudera.com> AuthorDate: Wed Jan 30 07:01:33 2019 -0800 IMPALA-8147: part 1/2: make make_*.sh redundant. The logic in that file, which is mostly about constructing argument lists for CMake and make, is moved to functions in buildall.sh. A new option -release_and_debug is added to buildall.sh to build both the debug and release builds. This is convenient for building a binary Impala for distribution because you want to have both sets of binaries available. make*.sh are not yet removed in order to make the transition easier. Testing: Ran buildall.sh locally with -release_and_debug, confirmed that all of the right binaries were generated. Change-Id: I70e4f65712166348ca006bc68e1a1e18e853d3a0 Reviewed-on: http://gerrit.cloudera.org:8080/12368 Reviewed-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com> --- be/src/common/CMakeLists.txt | 4 ++ bin/distcc/README.md | 7 +-- bin/make_impala.sh | 3 ++ buildall.sh | 100 +++++++++++++++++++++++++++++++++++-------- 4 files changed, 89 insertions(+), 25 deletions(-) diff --git a/be/src/common/CMakeLists.txt b/be/src/common/CMakeLists.txt index daa274d..2ea3b0c 100644 --- a/be/src/common/CMakeLists.txt +++ b/be/src/common/CMakeLists.txt @@ -36,12 +36,16 @@ add_library(Common ${VERSION_CC_GEN_OUTPUT} ) +# Command to generate the build version file if not present. We don't automatically +# regenerate the file if present, which speeds up incremental builds but can lead +# to the version being stale. add_custom_command( OUTPUT ${VERSION_CC_GEN_OUTPUT} COMMAND $ENV{IMPALA_HOME}/bin/gen_build_version.py COMMENT "Generating the version.cc file" VERBATIM ) + add_dependencies(Common gen-deps) add_library(GlobalFlags diff --git a/bin/distcc/README.md b/bin/distcc/README.md index a8487b2..180df0d 100644 --- a/bin/distcc/README.md +++ b/bin/distcc/README.md @@ -56,10 +56,6 @@ you can either ``` make -j$(distcc -j) ``` -or -``` -bin/make_impala.sh -``` ### Switching back to local compilation If you want to compile a very small change, a local build might be faster. @@ -71,8 +67,7 @@ to switch back switch_compiler distcc ``` ### Second time -If you open a new terminal and attempt to build with "make" or "bin/make_impala.sh", -that will fail. To fix: +If you open a new terminal and attempt to build with "make" that will fail. To fix: ``` source "$IMPALA_HOME"/bin/impala-config.sh # Skip if already done source "$IMPALA_HOME"/bin/distcc/distcc_env.sh diff --git a/bin/make_impala.sh b/bin/make_impala.sh index 8ac7639..2ce837f 100755 --- a/bin/make_impala.sh +++ b/bin/make_impala.sh @@ -103,6 +103,9 @@ do esac done +echo "IMPALA-8147: make_impala.sh, make_debug.sh, make_release.sh and make_asan.sh " +echo "are deprecated. Please migrate your scripts to use buildall.sh or make directly" + if [ $BUILD_TESTS -eq 1 ]; then MAKE_TARGETS+=" be-test be-benchmarks" fi diff --git a/buildall.sh b/buildall.sh index bfe3507..8a182a6 100755 --- a/buildall.sh +++ b/buildall.sh @@ -60,17 +60,19 @@ START_IMPALA_CLUSTER=0 IMPALA_KERBERIZE=0 SNAPSHOT_FILE= METASTORE_SNAPSHOT_FILE= -MAKE_IMPALA_ARGS="" CODE_COVERAGE=0 BUILD_ASAN=0 BUILD_FE_ONLY=0 +BUILD_TESTS=1 +GEN_CMAKE_ONLY=0 +BUILD_RELEASE_AND_DEBUG=0 BUILD_TIDY=0 BUILD_UBSAN=0 BUILD_UBSAN_FULL=0 BUILD_TSAN=0 +BUILD_SHARED_LIBS=0 # Export MAKE_CMD so it is visible in scripts that invoke make, e.g. copy-udfs-udas.sh export MAKE_CMD=make -LZO_CMAKE_ARGS= # Defaults that can be picked up from the environment, but are overridable through the # commandline. @@ -91,11 +93,11 @@ do TESTS_ACTION=0 ;; -build_shared_libs|-so) - MAKE_IMPALA_ARGS="${MAKE_IMPALA_ARGS} -build_shared_libs" + BUILD_SHARED_LIBS=1 ;; -notests) TESTS_ACTION=0 - MAKE_IMPALA_ARGS="${MAKE_IMPALA_ARGS} -notests" + BUILD_TESTS=0 ;; -format) FORMAT_CLUSTER=1 @@ -114,6 +116,9 @@ do -release) CMAKE_BUILD_TYPE=Release ;; + -release_and_debug) + BUILD_RELEASE_AND_DEBUG=1 + ;; -codecoverage) CODE_COVERAGE=1 ;; @@ -183,12 +188,10 @@ do BUILD_FE_ONLY=1 ;; -ninja) - MAKE_IMPALA_ARGS+=" -ninja" - LZO_CMAKE_ARGS+=" -GNinja" MAKE_CMD=ninja ;; -cmake_only) - MAKE_IMPALA_ARGS+=" -cmake_only" + GEN_CMAKE_ONLY=1 ;; -help|*) echo "buildall.sh - Builds Impala and runs all tests." @@ -199,6 +202,8 @@ do echo "[-format_cluster] : Format the minicluster [Default: False]" echo "[-format_metastore] : Format the metastore db [Default: False]" echo "[-format_sentry_policy_db] : Format the Sentry policy db [Default: False]" + echo "[-release_and_debug] : Build both release and debug binaries. Overrides "\ + "other build types [Default: false]" echo "[-release] : Release build [Default: debug]" echo "[-codecoverage] : Build with code coverage [Default: False]" echo "[-asan] : Address sanitizer build [Default: False]" @@ -297,8 +302,6 @@ if [[ ${BUILD_TSAN} -eq 1 ]]; then CMAKE_BUILD_TYPE=TSAN fi -MAKE_IMPALA_ARGS+=" -build_type=${CMAKE_BUILD_TYPE}" - # If we aren't kerberized then we certainly don't need to talk about # re-sourcing impala-config. if [[ ${IMPALA_KERBERIZE} -eq 0 ]]; then @@ -370,21 +373,67 @@ bootstrap_dependencies() { # Build the Impala frontend and its dependencies. build_fe() { - "$IMPALA_HOME/bin/make_impala.sh" ${MAKE_IMPALA_ARGS} -fe_only + generate_cmake_files $CMAKE_BUILD_TYPE + ${MAKE_CMD} ${IMPALA_MAKE_FLAGS} fe } -# Build all components. +# Build all components. The build type is specified as the first argument, and the +# second argument is 0 if targets that are independent of the build type (like the +# frontend) should not be built or non-zero otherwise. E.g. to build DEBUG including +# build-type-independent artifacts. +# build_all_components DEBUG 1 build_all_components() { + build_type=$1 + build_independent_targets=$2 echo ">>> Building all components" - # Build the Impala frontend, backend and external data source API. - MAKE_IMPALA_ARGS+=" -fe -cscope -tarballs" - if [[ -e "$IMPALA_LZO" ]] - then - MAKE_IMPALA_ARGS+=" -impala-lzo" + generate_cmake_files $build_type + + # Force regenerating the build version and timestamp (this doesn't happen automatically + # in incremental builds). + $IMPALA_HOME/bin/gen_build_version.py + + # If we skip specifying targets, everything we need gets built. + local MAKE_TARGETS="" + if [[ $BUILD_TESTS -eq 0 ]]; then + # Specify all the non-test targets + MAKE_TARGETS="impalad statestored catalogd fesupport loggingsupport ImpalaUdf \ + udasample udfsample" + if (( build_independent_targets )); then + MAKE_TARGETS+=" cscope fe tarballs" + fi + if [[ -e "$IMPALA_LZO" ]]; then + MAKE_TARGETS+=" impala-lzo" + fi fi + ${MAKE_CMD} -j${IMPALA_BUILD_THREADS:-4} ${IMPALA_MAKE_FLAGS} ${MAKE_TARGETS} +} + - echo "Running make_impala.sh ${MAKE_IMPALA_ARGS}" - "$IMPALA_HOME/bin/make_impala.sh" ${MAKE_IMPALA_ARGS} +# Called with the CMAKE_BUILD_TYPE as the first argument, e.g. +# generate_cmake_files DEBUG +generate_cmake_files() { + local build_type=$1 + echo ">>> Generating CMake files" "CMAKE_BUILD_TYPE=$build_type"\ + "BUILD_SHARED_LIBS=$BUILD_SHARED_LIBS" "MAKE_CMD=$MAKE_CMD" + # Remove cache to ensure that any changes to cmake arguments take effect. + rm -f ./CMakeCache.txt + local CMAKE_ARGS=(-DCMAKE_BUILD_TYPE=${build_type}) + if [[ $BUILD_SHARED_LIBS -eq 1 ]]; then + CMAKE_ARGS+=(-DBUILD_SHARED_LIBS=ON) + fi + if [[ "${MAKE_CMD}" = "ninja" ]]; then + CMAKE_ARGS+=(-GNinja) + fi + if [[ ("$build_type" == "ADDRESS_SANITIZER") \ + || ("$build_type" == "TIDY") \ + || ("$build_type" == "UBSAN") \ + || ("$build_type" == "UBSAN_FULL") \ + || ("$build_type" == "TSAN") ]]; then + CMAKE_ARGS+=(-DCMAKE_TOOLCHAIN_FILE=$IMPALA_HOME/cmake_modules/clang_toolchain.cmake) + else + CMAKE_ARGS+=(-DCMAKE_TOOLCHAIN_FILE=$IMPALA_HOME/cmake_modules/toolchain.cmake) + fi + cmake . ${CMAKE_ARGS[@]} } # Do any configuration of the test cluster required by the script arguments. @@ -492,7 +541,20 @@ if [[ "$BUILD_FE_ONLY" -eq 1 ]]; then exit 0 fi -build_all_components +if [[ "$GEN_CMAKE_ONLY" -eq 1 ]]; then + generate_cmake_files $CMAKE_BUILD_TYPE + exit 0 +fi +if [[ "$BUILD_RELEASE_AND_DEBUG" -eq 1 ]]; then + # Build the standard release and debug builds. We can't do this for arbitrary build + # types because many build types reuse the same be/build/debug and be/build/release + # trees. + build_all_components RELEASE 1 + # Avoid rebuilding targets that are independent of the build type. + build_all_components DEBUG 0 +else + build_all_components $CMAKE_BUILD_TYPE 1 +fi if [[ $NEED_MINICLUSTER -eq 1 ]]; then reconfigure_test_cluster