kevinw66 commented on code in PR #8710: URL: https://github.com/apache/incubator-gluten/pull/8710#discussion_r1959555456
########## ep/build-velox/src/setup-openeuler24.sh: ########## @@ -0,0 +1,199 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script documents setting up a openEuler24 host for Velox +# development. Running it should make you ready to compile. +# +# Environment variables: +# * INSTALL_PREREQUISITES="N": Skip installation of packages for build. +# * PROMPT_ALWAYS_RESPOND="n": Automatically respond to interactive prompts. +# Use "n" to never wipe directories. +# +# You can also run individual functions below by specifying them as arguments: +# $ ./setup-openeuler24.sh install_googletest install_fmt +# + +set -efx -o pipefail +# Some of the packages must be build with the same compiler flags +# so that some low level types are the same size. Also, disable warnings. +SCRIPTDIR=./scripts +source $SCRIPTDIR/setup-helper-functions.sh +CPU_TARGET="${CPU_TARGET:-avx}" +NPROC=$(getconf _NPROCESSORS_ONLN) +export CFLAGS=$(get_cxx_flags $CPU_TARGET) # Used by LZO. +export CXXFLAGS=$CFLAGS # Used by boost. +export CPPFLAGS=$CFLAGS # Used by LZO. +CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" +VELOX_BUILD_SHARED=${VELOX_BUILD_SHARED:-"OFF"} #Build folly and gflags shared for use in libvelox.so. +BUILD_DUCKDB="${BUILD_DUCKDB:-true}" +VERSION=$(cat /etc/os-release | grep VERSION_ID) +export INSTALL_PREFIX=${INSTALL_PREFIX:-"/usr/local"} +DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download} + +FB_OS_VERSION="v2024.07.01.00" +FMT_VERSION="10.1.1" +BOOST_VERSION="boost-1.84.0" +DUCKDB_VERSION="v0.8.1" + +function dnf_install { + dnf install -y -q --setopt=install_weak_deps=False "$@" +} + +# Install packages required for build. +function install_build_prerequisites { + dnf update -y + dnf_install dnf-plugins-core # For ccache, ninja + dnf update -y + dnf_install ninja-build cmake ccache gcc g++ git wget which patch + dnf_install autoconf automake python3-devel python3-pip libtool + + pip install cmake==3.28.3 +} + +# Install dependencies from the package managers. +function install_velox_deps_from_dnf { + dnf_install libevent-devel \ + openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel \ + libdwarf-devel elfutils-libelf-devel curl-devel libicu-devel bison flex \ + libsodium-devel zlib-devel + + # install sphinx for doc gen + pip install sphinx sphinx-tabs breathe sphinx_rtd_theme +} + +function install_gflags { + # Remove an older version if present. + dnf remove -y gflags + wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags + cmake_install_dir gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 +} + +function install_glog { + wget_and_untar https://github.com/google/glog/archive/v0.6.0.tar.gz glog + cmake_install_dir glog -DBUILD_SHARED_LIBS=ON +} + +function install_lzo { + wget_and_untar http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz lzo + ( + cd ${DEPENDENCY_DIR}/lzo + ./configure --prefix=${INSTALL_PREFIX} --enable-shared --disable-static --docdir=/usr/share/doc/lzo-2.10 + make "-j${NPROC}" + make install + ) +} + +function install_boost { + wget_and_untar https://github.com/boostorg/boost/releases/download/${BOOST_VERSION}/${BOOST_VERSION}.tar.gz boost + ( + cd ${DEPENDENCY_DIR}/boost + ./bootstrap.sh --prefix=${INSTALL_PREFIX} + ${SUDO} ./b2 "-j${NPROC}" -d0 install threading=multi --without-python + ) +} + +function install_snappy { + wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy + cmake_install_dir snappy -DSNAPPY_BUILD_TESTS=OFF +} + +function install_fmt { + wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt + cmake_install_dir fmt -DFMT_TEST=OFF +} + +function install_protobuf { + wget_and_untar https://github.com/protocolbuffers/protobuf/releases/download/v21.8/protobuf-all-21.8.tar.gz protobuf + ( + cd ${DEPENDENCY_DIR}/protobuf + ./configure CXXFLAGS="-fPIC" --prefix=${INSTALL_PREFIX} + make "-j${NPROC}" + make install + ldconfig + ) +} + +function install_fizz { + wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz + cmake_install_dir fizz/fizz -DBUILD_TESTS=OFF +} + +function install_folly { + wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly + cmake_install_dir folly -DBUILD_SHARED_LIBS="$VELOX_BUILD_SHARED" -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON +} + +function install_wangle { + wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle + cmake_install_dir wangle/wangle -DBUILD_TESTS=OFF +} + +function install_fbthrift { + wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift + cmake_install_dir fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF +} + +function install_mvfst { + wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst + cmake_install_dir mvfst -DBUILD_TESTS=OFF +} + +function install_duckdb { + if $BUILD_DUCKDB ; then + echo 'Building DuckDB' + wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/${DUCKDB_VERSION}.tar.gz duckdb + cmake_install_dir duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release + fi +} + +function install_velox_deps { + run_and_time install_velox_deps_from_dnf + run_and_time install_gflags + run_and_time install_glog + run_and_time install_lzo + run_and_time install_snappy + run_and_time install_boost + run_and_time install_protobuf + run_and_time install_fmt + run_and_time install_folly + run_and_time install_fizz + run_and_time install_wangle + run_and_time install_mvfst + run_and_time install_fbthrift + run_and_time install_duckdb +} + +(return 2> /dev/null) && return # If script was sourced, don't run commands. + +( + if [[ $# -ne 0 ]]; then + for cmd in "$@"; do + run_and_time "${cmd}" + done + echo "All specified dependencies installed!" + else + if [ "${INSTALL_PREREQUISITES:-Y}" == "Y" ]; then + echo "Installing build dependencies" + run_and_time install_build_prerequisites + else + echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set" + fi + + install_velox_deps + echo "All dependencies for Velox installed!" + dnf clean all + fi +) Review Comment: Done -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
