This is an automated email from the ASF dual-hosted git repository.
philo-he pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 6f508d2962 [MINOR][GLUTEN-12137] Remove unmaintained gluten-te folder
(#12149)
6f508d2962 is described below
commit 6f508d2962737263ad03f4e36e2c147fde45d186
Author: Philo He <[email protected]>
AuthorDate: Wed May 27 01:29:58 2026 -0700
[MINOR][GLUTEN-12137] Remove unmaintained gluten-te folder (#12149)
---
CONTRIBUTING.md | 2 -
tools/gluten-te/.fixme | 1 -
tools/gluten-te/centos/build.sh | 93 ------------
tools/gluten-te/centos/buildenv.sh | 81 -----------
tools/gluten-te/centos/cbash-build.sh | 71 ---------
tools/gluten-te/centos/cbash-mount.sh | 60 --------
tools/gluten-te/centos/centos-7-deps.sh | 105 -------------
tools/gluten-te/centos/centos-8-deps.sh | 36 -----
tools/gluten-te/centos/defaults.conf | 66 ---------
tools/gluten-te/centos/dockerfile-build | 106 --------------
tools/gluten-te/centos/dockerfile-buildenv | 89 -----------
.../buildhere-veloxbe-portable-libs/README.md | 33 -----
.../buildhere-veloxbe-portable-libs/run-default.sh | 21 ---
.../buildhere-veloxbe-portable-libs/run.sh | 29 ----
.../buildhere-veloxbe-portable-libs/scripts/all.sh | 52 -------
.../gluten-te/centos/gha/gha-checkout/checkout.sh | 49 -------
tools/gluten-te/centos/gha/gha-checkout/clean.sh | 25 ----
tools/gluten-te/centos/gha/gha-checkout/cp.sh | 29 ----
tools/gluten-te/centos/gha/gha-checkout/exec.sh | 28 ----
.../centos/gha/gha-checkout/scripts/init.sh | 25 ----
tools/gluten-te/centos/shared.sh | 48 ------
tools/gluten-te/ubuntu/README.md | 106 --------------
tools/gluten-te/ubuntu/build.sh | 92 ------------
tools/gluten-te/ubuntu/buildenv.sh | 80 ----------
tools/gluten-te/ubuntu/cbash-build.sh | 71 ---------
tools/gluten-te/ubuntu/cbash-mount.sh | 60 --------
tools/gluten-te/ubuntu/defaults.conf | 94 ------------
tools/gluten-te/ubuntu/docker-compose.yml | 16 --
tools/gluten-te/ubuntu/dockerfile-build | 106 --------------
tools/gluten-te/ubuntu/dockerfile-buildenv | 162 ---------------------
tools/gluten-te/ubuntu/dockerfile-tpc | 62 --------
.../ubuntu/examples/buildhere-veloxbe-dev/run.sh | 27 ----
.../examples/buildhere-veloxbe-dev/scripts/all.sh | 42 ------
.../buildhere-veloxbe-portable-libs/README.md | 33 -----
.../buildhere-veloxbe-portable-libs/run-default.sh | 21 ---
.../buildhere-veloxbe-portable-libs/run.sh | 29 ----
.../buildhere-veloxbe-portable-libs/scripts/all.sh | 51 -------
.../ubuntu/examples/buildhere-veloxbe/run.sh | 29 ----
.../gluten-te/ubuntu/gha/gha-checkout/checkout.sh | 49 -------
tools/gluten-te/ubuntu/gha/gha-checkout/clean.sh | 25 ----
tools/gluten-te/ubuntu/gha/gha-checkout/cp.sh | 29 ----
tools/gluten-te/ubuntu/gha/gha-checkout/exec.sh | 28 ----
.../ubuntu/gha/gha-checkout/scripts/init.sh | 25 ----
tools/gluten-te/ubuntu/scripts/cmd.sh | 18 ---
tools/gluten-te/ubuntu/scripts/set-login-env.sh | 18 ---
tools/gluten-te/ubuntu/shared.sh | 48 ------
tools/gluten-te/ubuntu/tpc.sh | 154 --------------------
47 files changed, 2524 deletions(-)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index fe06148153..8aa3d26814 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -79,8 +79,6 @@ To check CI failure for Velox backend, please go into the
GitHub action page fro
To see the perf. impact on Velox backend, you can comment `/Benchmark Velox`
on PR page to trigger a pretest. The benchmark
(currently TPC-H) result will be posted after completed.
-If some new dependency is required to be installed, you may need to do some
change for CI docker at [this
folder](https://github.com/apache/gluten/tree/main/tools/gluten-te).
-
### Code Review
Please ensure no CI failure is reported for your PR.
diff --git a/tools/gluten-te/.fixme b/tools/gluten-te/.fixme
deleted file mode 100644
index 97143f47ff..0000000000
--- a/tools/gluten-te/.fixme
+++ /dev/null
@@ -1 +0,0 @@
-FIXME: Consolidate /opt/gluten/tools/gluten-te/centos/gha/gha-checkout with
/opt/gluten/tools/gluten-te/ubuntu/gha/gha-checkout
diff --git a/tools/gluten-te/centos/build.sh b/tools/gluten-te/centos/build.sh
deleted file mode 100755
index 52e8efff64..0000000000
--- a/tools/gluten-te/centos/build.sh
+++ /dev/null
@@ -1,93 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-BASEDIR=$(dirname $0)
-
-source "$BASEDIR/buildenv.sh"
-
-## Debug build flags
-
-# Create debug build
-DEBUG_BUILD=${DEBUG_BUILD:-$DEFAULT_DEBUG_BUILD}
-
-if [ -n $JDK_DEBUG_BUILD ]
-then
- echo "Do not set JDK_DEBUG_BUILD manually!"
-fi
-
-if [ -n $GLUTEN_DEBUG_BUILD ]
-then
- echo "Do not set GLUTEN_DEBUG_BUILD manually!"
-fi
-
-if [ "$DEBUG_BUILD" == "ON" ]
-then
- JDK_DEBUG_BUILD=OFF
- GLUTEN_DEBUG_BUILD=ON
-else
- JDK_DEBUG_BUILD=OFF
- GLUTEN_DEBUG_BUILD=OFF
-fi
-
-# The branches used to prepare dependencies
-CACHE_GLUTEN_REPO=${CACHE_GLUTEN_REPO:-$DEFAULT_GLUTEN_REPO}
-CACHE_GLUTEN_BRANCH=${CACHE_GLUTEN_BRANCH:-$DEFAULT_GLUTEN_BRANCH}
-
-CACHE_GLUTEN_COMMIT="$(git ls-remote $CACHE_GLUTEN_REPO $CACHE_GLUTEN_BRANCH |
awk '{print $1;}')"
-
-if [ -z "$CACHE_GLUTEN_COMMIT" ]
-then
- echo "Unable to parse CACHE_GLUTEN_COMMIT."
- exit 1
-fi
-
-# Backend type
-BUILD_BACKEND_TYPE=${BUILD_BACKEND_TYPE:-$DEFAULT_BUILD_BACKEND_TYPE}
-
-##
-
-BUILD_DOCKER_BUILD_ARGS=
-
-BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --ulimit nofile=8192:8192"
-BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --build-arg
BUILDKIT_INLINE_CACHE=1"
-BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --build-arg
DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE=$DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE"
-BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --build-arg
JDK_DEBUG_BUILD=$JDK_DEBUG_BUILD"
-BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --build-arg
GLUTEN_DEBUG_BUILD=$GLUTEN_DEBUG_BUILD"
-BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --build-arg
CACHE_GLUTEN_REPO=$CACHE_GLUTEN_REPO"
-BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --build-arg
CACHE_GLUTEN_COMMIT=$CACHE_GLUTEN_COMMIT"
-BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --build-arg
BUILD_BACKEND_TYPE=$BUILD_BACKEND_TYPE"
-BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS -f $BASEDIR/dockerfile-build"
-BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --target gluten-build"
-BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS -t
$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE"
-
-if [ -n "$DOCKER_CACHE_REGISTRY" ]
-then
- BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --cache-from
$DOCKER_CACHE_REGISTRY/$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE"
-fi
-
-BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS $BASEDIR"
-
-docker build $BUILD_DOCKER_BUILD_ARGS
-
-if [ -n "$DOCKER_PUSH_REGISTRY" ]
-then
- docker tag "$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE"
"$DOCKER_PUSH_REGISTRY/$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE"
- docker push "$DOCKER_PUSH_REGISTRY/$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE"
-fi
-
-# EOF
diff --git a/tools/gluten-te/centos/buildenv.sh
b/tools/gluten-te/centos/buildenv.sh
deleted file mode 100755
index 70314c9084..0000000000
--- a/tools/gluten-te/centos/buildenv.sh
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-BASEDIR=$(dirname $0)
-
-source "$BASEDIR/shared.sh"
-
-# Docker registry used to pull layer cache to speed-up builds
-DOCKER_CACHE_REGISTRY=${DOCKER_CACHE_REGISTRY:-$DEFAULT_DOCKER_CACHE_REGISTRY}
-
-# Docker registry to push pre-built images
-DOCKER_PUSH_REGISTRY=${DOCKER_PUSH_REGISTRY:-$DEFAULT_DOCKER_PUSH_REGISTRY}
-
-# HTTP proxy
-HTTP_PROXY_HOST=${HTTP_PROXY_HOST:-$DEFAULT_HTTP_PROXY_HOST}
-HTTP_PROXY_PORT=${HTTP_PROXY_PORT:-$DEFAULT_HTTP_PROXY_PORT}
-
-# If on, use maven mirror settings for PRC's network environment
-USE_ALI_MAVEN_MIRROR=${USE_ALI_MAVEN_MIRROR:-$DEFAULT_USE_ALI_MAVEN_MIRROR}
-
-# Whether to build Spark binaries in buildenv image
-BUILD_SPARK_BINARIES=${BUILD_SPARK_BINARIES:-$DEFAULT_BUILD_SPARK_BINARIES}
-
-# Set timezone name
-TIMEZONE=${TIMEZONE:-$DEFAULT_TIMEZONE}
-
-
-if [ "$USE_ALI_MAVEN_MIRROR" == "ON" ]
-then
- MAVEN_MIRROR_URL='https://maven.aliyun.com/repository/public'
-else
- MAVEN_MIRROR_URL=
-fi
-
-##
-
-BUILDENV_DOCKER_BUILD_ARGS=
-
-BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --ulimit
nofile=8192:8192"
-BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --build-arg
BUILDKIT_INLINE_CACHE=1"
-BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --build-arg
TIMEZONE=$TIMEZONE"
-BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --build-arg
OS_IMAGE_NAME=$OS_IMAGE_NAME --build-arg OS_IMAGE_TAG=$OS_IMAGE_TAG"
-BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --build-arg
MAVEN_MIRROR_URL=$MAVEN_MIRROR_URL"
-BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --build-arg
HTTP_PROXY_HOST=$HTTP_PROXY_HOST"
-BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --build-arg
HTTP_PROXY_PORT=$HTTP_PROXY_PORT"
-BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --build-arg
BUILD_SPARK_BINARIES=$BUILD_SPARK_BINARIES"
-BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS -f
$BASEDIR/dockerfile-buildenv"
-BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --target
gluten-buildenv"
-BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS -t
$DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE"
-
-if [ -n "$DOCKER_CACHE_REGISTRY" ]
-then
- BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --cache-from
$DOCKER_CACHE_REGISTRY/$DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE"
-fi
-
-BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS $BASEDIR"
-
-docker build $BUILDENV_DOCKER_BUILD_ARGS
-
-if [ -n "$DOCKER_PUSH_REGISTRY" ]
-then
- docker tag "$DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE"
"$DOCKER_PUSH_REGISTRY/$DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE"
- docker push
"$DOCKER_PUSH_REGISTRY/$DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE"
-fi
-
-# EOF
diff --git a/tools/gluten-te/centos/cbash-build.sh
b/tools/gluten-te/centos/cbash-build.sh
deleted file mode 100755
index f3c5c1e734..0000000000
--- a/tools/gluten-te/centos/cbash-build.sh
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-BASEDIR=$(dirname $0)
-
-source "$BASEDIR/shared.sh"
-
-# Docker registry used to pull layer cache to speed-up builds
-DOCKER_PULL_REGISTRY=${DOCKER_PULL_REGISTRY:-$DEFAULT_DOCKER_PULL_REGISTRY}
-
-# Non-interactive during docker run
-NON_INTERACTIVE=${NON_INTERACTIVE:-$DEFAULT_NON_INTERACTIVE}
-
-# Do not remove stopped docker container
-PRESERVE_CONTAINER=${PRESERVE_CONTAINER:-$DEFAULT_PRESERVE_CONTAINER}
-
-# Docker options
-EXTRA_DOCKER_OPTIONS=${EXTRA_DOCKER_OPTIONS:-$DEFAULT_EXTRA_DOCKER_OPTIONS}
-
-# Whether to mount Maven cache
-MOUNT_MAVEN_CACHE=${MOUNT_MAVEN_CACHE:-$DEFAULT_MOUNT_MAVEN_CACHE}
-
-if [ -n "$DOCKER_PULL_REGISTRY" ]
-then
- docker pull $DOCKER_PULL_REGISTRY/$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE
- docker tag $DOCKER_PULL_REGISTRY/$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE
$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE
-else
- $BASEDIR/build.sh
-fi
-
-CBASH_DOCKER_RUN_ARGS=
-if [ "$NON_INTERACTIVE" != "ON" ]
-then
- CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS -it"
-fi
-if [ "$PRESERVE_CONTAINER" != "ON" ]
-then
- CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --rm"
-fi
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --init"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --privileged"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --ulimit nofile=65536:65536"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --ulimit core=-1"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --security-opt
seccomp=unconfined"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS -e NUM_THREADS=$(nproc)"
-if [ "$MOUNT_MAVEN_CACHE" == "ON" ]
-then
- CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS -v
$HOME/.m2/repository:/root/.m2/repository"
-fi
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS -v $HOME/.ccache:/root/.ccache"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS $EXTRA_DOCKER_OPTIONS"
-
-CBASH_BASH_ARGS="$*"
-BASH_ARGS="$CBASH_BASH_ARGS"
-
-docker run $CBASH_DOCKER_RUN_ARGS $DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE
bash -c "cd /opt/gluten && $BASH_ARGS"
diff --git a/tools/gluten-te/centos/cbash-mount.sh
b/tools/gluten-te/centos/cbash-mount.sh
deleted file mode 100755
index a09a748d11..0000000000
--- a/tools/gluten-te/centos/cbash-mount.sh
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-BASEDIR=$(dirname $0)
-
-source "$BASEDIR/buildenv.sh"
-
-# Non-interactive during docker run
-NON_INTERACTIVE=${NON_INTERACTIVE:-$DEFAULT_NON_INTERACTIVE}
-
-# Do not remove stopped docker container
-PRESERVE_CONTAINER=${PRESERVE_CONTAINER:-$DEFAULT_PRESERVE_CONTAINER}
-
-# Docker options
-EXTRA_DOCKER_OPTIONS=${EXTRA_DOCKER_OPTIONS:-$DEFAULT_EXTRA_DOCKER_OPTIONS}
-
-# Whether to mount Maven cache
-MOUNT_MAVEN_CACHE=${MOUNT_MAVEN_CACHE:-$DEFAULT_MOUNT_MAVEN_CACHE}
-
-CBASH_DOCKER_RUN_ARGS=
-if [ "$NON_INTERACTIVE" != "ON" ]
-then
- CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS -it"
-fi
-if [ "$PRESERVE_CONTAINER" != "ON" ]
-then
- CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --rm"
-fi
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --init"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --privileged"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --ulimit nofile=65536:65536"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --ulimit core=-1"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --security-opt
seccomp=unconfined"
-if [ "$MOUNT_MAVEN_CACHE" == "ON" ]
-then
- CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS -v
$HOME/.m2/repository:/root/.m2/repository"
-fi
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS -v $HOME/.ccache:/root/.ccache"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS -v $PWD:/opt/gluten"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS $EXTRA_DOCKER_OPTIONS"
-
-CBASH_BASH_ARGS="$*"
-BASH_ARGS="$CBASH_BASH_ARGS"
-
-docker run $CBASH_DOCKER_RUN_ARGS $DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE
bash -c "cd /opt/gluten && $BASH_ARGS"
diff --git a/tools/gluten-te/centos/centos-7-deps.sh
b/tools/gluten-te/centos/centos-7-deps.sh
deleted file mode 100755
index b2d26b2585..0000000000
--- a/tools/gluten-te/centos/centos-7-deps.sh
+++ /dev/null
@@ -1,105 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-sed -i -e "s/enabled=1/enabled=0/" /etc/yum/pluginconf.d/fastestmirror.conf
-sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
-sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-*
-yum install -y centos-release-scl
-rm -f /etc/yum.repos.d/CentOS-SCLo-scl.repo
-sed -i \
- -e 's/^mirrorlist/#mirrorlist/' \
- -e 's/^#baseurl/baseurl/' \
- -e 's/mirror\.centos\.org/vault.centos.org/' \
- /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo
-
-yum -y install epel-release
-yum -y install \
- wget curl tar zip unzip which patch sudo \
- ninja-build perl-IPC-Cmd autoconf autoconf-archive automake libtool \
- devtoolset-11 python3 pip dnf \
- bison \
- java-1.8.0-openjdk java-1.8.0-openjdk-devel
-
-# Link c++ to the one in devtoolset.
-ln -s /opt/rh/devtoolset-11/root/usr/bin/c++ /usr/bin/c++
-
-semver() {
- echo "$@" | awk -F. '{ printf("%d%05d%05d", $1,$2,$3); }'
-}
-
-pip3 install --upgrade pip
-
-# cmake >= 3.28.3
-pip3 install cmake==3.28.3
-
-# git >= 2.7.4
-if [[ "$(git --version)" != "git version 2."* ]]; then
- [ -f /etc/yum.repos.d/ius.repo ] || yum -y install
https://repo.ius.io/ius-release-el7.rpm
- yum -y remove git
- yum -y install git236
-fi
-
-# flex>=2.6.0
-if [[ "$(PATH="/usr/local/bin:$PATH" flex --version 2>&1)" != "flex 2.6."* ]];
then
- yum -y install gettext-devel
-
FLEX_URL="https://github.com/westes/flex/releases/download/v2.6.4/flex-2.6.4.tar.gz";
- mkdir -p /tmp/flex
- wget -q --max-redirect 3 -O - "${FLEX_URL}" | tar -xz -C /tmp/flex
--strip-components=1
- cd /tmp/flex
- ./autogen.sh
- ./configure
- make install
- cd
- rm -rf /tmp/flex
-fi
-
-# automake>=1.14
-installed_automake_version="$(aclocal --version | sed -En "1s/^.*
([1-9\.]*)$/\1/p")"
-if [ "$(semver "$installed_automake_version")" -lt "$(semver 1.14)" ]; then
- mkdir -p /tmp/automake
- AUTOMAKE_URL1="https://ftp.gnu.org/gnu/automake/automake-1.16.5.tar.xz";
-
AUTOMAKE_URL2="https://www.mirrorservice.org/sites/ftp.gnu.org/gnu/automake/automake-1.16.5.tar.xz";
- wget -O /tmp/automake.tar.xz "$AUTOMAKE_URL1" || wget -O
/tmp/automake.tar.xz "$AUTOMAKE_URL2"
- tar -xf /tmp/automake.tar.xz -C /tmp/automake --strip-components=1
- rm -f /tmp/automake.tar.xz
- cd /tmp/automake
- ./configure
- make install -j
- cd
- rm -rf /tmp/automake
-
- # Fix aclocal search path
- echo /usr/share/aclocal >/usr/local/share/aclocal/dirlist
-fi
-
-# cmake
-if [ -z "$(which mvn)" ]; then
- maven_version=3.9.2
- maven_install_dir=/opt/maven-$maven_version
- if [ -d /opt/maven-$maven_version ]; then
- echo "Failed to install maven: ${maven_install_dir} is exists" >&2
- exit 1
- fi
-
- cd /tmp
- wget
https://archive.apache.org/dist/maven/maven-3/$maven_version/binaries/apache-maven-$maven_version-bin.tar.gz
- tar -xvf apache-maven-$maven_version-bin.tar.gz
- rm -f apache-maven-$maven_version-bin.tar.gz
- mv apache-maven-$maven_version "${maven_install_dir}"
- ln -s "${maven_install_dir}/bin/mvn" /usr/local/bin/mvn
-fi
diff --git a/tools/gluten-te/centos/centos-8-deps.sh
b/tools/gluten-te/centos/centos-8-deps.sh
deleted file mode 100755
index 0031b7ae28..0000000000
--- a/tools/gluten-te/centos/centos-8-deps.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-# TODO, use CentOS 7 instead.
-echo " This script is out of date!"
-exit 1
-
-sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
-sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-*
-
-dnf install -y epel-release sudo
-yum -y update && yum clean all && yum install -y dnf-plugins-core
-yum config-manager --set-enabled powertools
-dnf --enablerepo=powertools install -y ninja-build
-dnf --enablerepo=powertools install -y libdwarf-devel
-dnf install -y --setopt=install_weak_deps=False ccache gcc-toolset-11 git wget
which libevent-devel \
- openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel \
- curl-devel cmake libicu-devel
-
-yum -y update && yum clean all && yum install -y java-1.8.0-openjdk-devel patch
-dnf -y install gcc-toolset-11-gcc gcc-toolset-11-gcc-c++
diff --git a/tools/gluten-te/centos/defaults.conf
b/tools/gluten-te/centos/defaults.conf
deleted file mode 100755
index b1338a6df3..0000000000
--- a/tools/gluten-te/centos/defaults.conf
+++ /dev/null
@@ -1,66 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-## For basic scripts
-
-# Non-interactive during docker run
-DEFAULT_NON_INTERACTIVE=OFF
-
-# Do not remove stopped docker container
-DEFAULT_PRESERVE_CONTAINER=OFF
-
-# The codes will be used in build
-DEFAULT_GLUTEN_REPO=https://github.com/apache/gluten.git
-DEFAULT_GLUTEN_BRANCH=main
-
-# Create debug build
-DEFAULT_DEBUG_BUILD=OFF
-
-# Backend type (velox)
-DEFAULT_BUILD_BACKEND_TYPE=velox
-
-# HTTP proxy
-# If http proxy is http://XXX.com:123, write it as
-# DEFAULT_HTTP_PROXY_HOST=xxx.com
-# DEFAULT_HTTP_PROXY_PORT=123
-# do not write http://
-DEFAULT_HTTP_PROXY_HOST=
-DEFAULT_HTTP_PROXY_PORT=
-
-# If on, use maven mirror settings for PRC's network environment
-DEFAULT_USE_ALI_MAVEN_MIRROR=OFF
-
-# Whether to build Spark binaries in buildenv image
-DEFAULT_BUILD_SPARK_BINARIES=OFF
-
-# Base operator system image used in build scripts.
-DEFAULT_OS_IMAGE_NAME=centos
-
-# Version ID of os image
-DEFAULT_OS_IMAGE_TAG=7
-
-# Set timezone name
-DEFAULT_TIMEZONE=Asia/Shanghai
-
-# Build will result in this image
-DEFAULT_DOCKER_TARGET_IMAGE_BUILDENV=gluten-te/gluten-buildenv
-
-# Build will result in this image
-DEFAULT_DOCKER_TARGET_IMAGE_BUILD=gluten-te/gluten-build
-
-# Docker registry to push pre-built images
-DEFAULT_DOCKER_PULL_REGISTRY=
-
-# Docker registry used to pull layer cache to speed-up builds
-DEFAULT_DOCKER_CACHE_REGISTRY=
-
-# Docker registry to push pre-built images
-DEFAULT_DOCKER_PUSH_REGISTRY=
-
-## For cbash.sh
-
-# Whether to mount Maven cache
-DEFAULT_MOUNT_MAVEN_CACHE=OFF
-
-# EOF
diff --git a/tools/gluten-te/centos/dockerfile-build
b/tools/gluten-te/centos/dockerfile-build
deleted file mode 100644
index 5292887c0b..0000000000
--- a/tools/gluten-te/centos/dockerfile-build
+++ /dev/null
@@ -1,106 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-ARG DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE
-
-FROM $DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE AS gluten-build
-MAINTAINER Hongze Zhang<[email protected]>
-
-# Whether debug build is enabled
-ARG JDK_DEBUG_BUILD
-ARG GLUTEN_DEBUG_BUILD
-RUN echo "JDK debug build is [$JDK_DEBUG_BUILD]!"
-RUN echo "Gluten debug build is [$GLUTEN_DEBUG_BUILD]!"
-
-# If JDK debug is on
-RUN if [ "$JDK_DEBUG_BUILD" == "ON" ]; \
- then \
- apt-get update; \
- DEBIAN_FRONTEND=noninteractive apt-get uninstall -y openjdk-8-jdk; \
- DEBIAN_FRONTEND=noninteractive apt-get uninstall -y maven; \
- mkdir -p /opt/jdk/ \
- && mkdir -p /opt/maven/ \
- && cd /opt/jdk/ \
- && wget
https://builds.shipilev.net/openjdk-jdk8/openjdk-jdk8-linux-x86_64-server-fastdebug-gcc8-glibc2.28.tar.xz
\
- && tar -xvf
openjdk-jdk8-linux-x86_64-server-fastdebug-gcc8-glibc2.28.tar.xz \
- && rm -f
openjdk-jdk8-linux-x86_64-server-fastdebug-gcc8-glibc2.28.tar.xz \
- && cd /opt/maven/ \
- && wget
https://dlcdn.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz
\
- && tar -xvf apache-maven-3.6.3-bin.tar.gz \
- && rm -f apache-maven-3.6.3-bin.tar.gz \
- && cp -rs /opt/jdk/j2sdk-image/bin/* /usr/local/bin/ \
- && cp -rs /opt/maven/apache-maven-3.6.3/bin/mvn /usr/local/bin/ \
- && echo "JAVA_HOME=/opt/jdk/j2sdk-image" > ~/.mavenrc; \
- fi
-
-# These branches are mainly for pre-downloading dependencies to speed-up
builds.
-# Thus it should not be required to change these values every time when the
build branch
-# is changed.
-ARG CACHE_GLUTEN_REPO
-ARG CACHE_GLUTEN_COMMIT
-
-RUN test -n "$CACHE_GLUTEN_REPO" || (echo "CACHE_GLUTEN_REPO not set" && false)
-RUN test -n "$CACHE_GLUTEN_COMMIT" || (echo "CACHE_GLUTEN_COMMIT not set" &&
false)
-
-RUN cd /opt \
- && git clone $CACHE_GLUTEN_REPO gluten \
- && cd gluten \
- && git fetch $CACHE_GLUTEN_REPO
$CACHE_GLUTEN_COMMIT:build_$CACHE_GLUTEN_COMMIT \
- && git checkout build_$CACHE_GLUTEN_COMMIT
-
-# Set ccache size
-RUN ccache -M 128G
-RUN ccache -s
-
-# Default Gluten Maven build options (empty as of now)
-ENV GLUTEN_MAVEN_OPTIONS=
-#RUN set-login-env "GLUTEN_MAVEN_OPTIONS="
-
-ARG BUILD_BACKEND_TYPE
-
-RUN test -n "$BUILD_BACKEND_TYPE" || (echo "BUILD_BACKEND_TYPE not set" &&
false)
-
-RUN if [ "$BUILD_BACKEND_TYPE" == "velox" ]; \
- then \
- if [ "$GLUTEN_DEBUG_BUILD" == "ON" ]; then GLUTEN_BUILD_TYPE="Debug";
else GLUTEN_BUILD_TYPE="Release"; fi; \
- DEPS_INSTALL_SCRIPT="source /env.sh && bash
/opt/gluten/dev/builddeps-veloxbe.sh \
- --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON
--enable_abfs=ON \
- --build_type=$GLUTEN_BUILD_TYPE"; \
- EXTRA_MAVEN_OPTIONS="-Pspark-3.5 \
- -Pbackends-velox \
- -Pceleborn \
- -Puniffle \
- -Piceberg \
- -Pdelta \
- -DskipTests \
- -Dscalastyle.skip=true \
- -Dcheckstyle.skip=true"; \
- else \
- echo "Unrecognizable backend type: $BUILD_BACKEND_TYPE"; \
- exit 1; \
- fi \
- && echo $EXTRA_MAVEN_OPTIONS > ~/.gluten-mvn-options \
- && echo $DEPS_INSTALL_SCRIPT > ~/.gluten-deps-install-script
-
-# Prebuild Gluten
-RUN EXTRA_MAVEN_OPTIONS=$(cat ~/.gluten-mvn-options) \
- DEPS_INSTALL_SCRIPT=$(cat ~/.gluten-deps-install-script) \
- && cd /opt/gluten \
- && bash -c "$DEPS_INSTALL_SCRIPT" \
- && bash -c "mvn clean install $GLUTEN_MAVEN_OPTIONS $EXTRA_MAVEN_OPTIONS"
-
-# EOF
diff --git a/tools/gluten-te/centos/dockerfile-buildenv
b/tools/gluten-te/centos/dockerfile-buildenv
deleted file mode 100755
index 4d9b7c67e0..0000000000
--- a/tools/gluten-te/centos/dockerfile-buildenv
+++ /dev/null
@@ -1,89 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-ARG OS_IMAGE_NAME
-ARG OS_IMAGE_TAG
-
-FROM $OS_IMAGE_NAME:$OS_IMAGE_TAG AS gluten-buildenv
-
-MAINTAINER Hongze Zhang<[email protected]>
-
-SHELL ["/bin/bash", "-c"]
-
-# REQUIRED PROXYS: WGET, GIT, MAVEN (also Maven mirror)
-ARG HTTP_PROXY_HOST
-ARG HTTP_PROXY_PORT
-
-ENV http_proxy=${HTTP_PROXY_HOST:+"http://$HTTP_PROXY_HOST:$HTTP_PROXY_PORT"}
-ENV https_proxy=${HTTP_PROXY_HOST:+"http://$HTTP_PROXY_HOST:$HTTP_PROXY_PORT"}
-
-ARG MAVEN_MIRROR_URL
-
-RUN if [ -n "$MAVEN_MIRROR_URL" ]; \
- then \
-
MAVEN_SETTINGS_TEMPLATE="<settings><mirrors><mirror><id>mavenmirror</id><mirrorOf>central</mirrorOf><name>MavenMirror</name><url>{{MAVEN_MIRROR_URL}}</url></mirror></mirrors><proxies><proxy><id>httpproxy</id><active>{{MAVEN_PROXY_ENABLE}}</active><protocol>http</protocol><host>{{MAVEN_PROXY_HOST}}</host><port>{{MAVEN_PROXY_PORT}}</port></proxy><proxy><id>httpsproxy</id><active>{{MAVEN_PROXY_ENABLE}}</active><protocol>https</protocol><host>{{MAVEN_PROXY_HOST}}</host><port>{{MAVEN_PR
[...]
- MAVEN_SETTINGS_TEMPLATE=$(echo $MAVEN_SETTINGS_TEMPLATE | sed
"s@{{MAVEN_MIRROR_URL}}@$MAVEN_MIRROR_URL@g"); \
- else \
-
MAVEN_SETTINGS_TEMPLATE="<settings><proxies><proxy><id>httpproxy</id><active>{{MAVEN_PROXY_ENABLE}}</active><protocol>http</protocol><host>{{MAVEN_PROXY_HOST}}</host><port>{{MAVEN_PROXY_PORT}}</port></proxy><proxy><id>httpsproxy</id><active>{{MAVEN_PROXY_ENABLE}}</active><protocol>https</protocol><host>{{MAVEN_PROXY_HOST}}</host><port>{{MAVEN_PROXY_PORT}}</port></proxy></proxies></settings>";
\
- fi \
- && if [ -n "$HTTP_PROXY_HOST" ]; \
- then \
- MAVEN_SETTINGS_TEMPLATE=$(echo $MAVEN_SETTINGS_TEMPLATE | sed
"s/{{MAVEN_PROXY_ENABLE}}/true/g"); \
- MAVEN_SETTINGS_TEMPLATE=$(echo $MAVEN_SETTINGS_TEMPLATE | sed
"s/{{MAVEN_PROXY_HOST}}/$HTTP_PROXY_HOST/g"); \
- MAVEN_SETTINGS_TEMPLATE=$(echo $MAVEN_SETTINGS_TEMPLATE | sed
"s/{{MAVEN_PROXY_PORT}}/$HTTP_PROXY_PORT/g"); \
- else \
- MAVEN_SETTINGS_TEMPLATE=$(echo $MAVEN_SETTINGS_TEMPLATE | sed
"s/{{MAVEN_PROXY_ENABLE}}/false/g"); \
- MAVEN_SETTINGS_TEMPLATE=$(echo $MAVEN_SETTINGS_TEMPLATE | sed
"s/{{MAVEN_PROXY_HOST}}/localhost/g"); \
- MAVEN_SETTINGS_TEMPLATE=$(echo $MAVEN_SETTINGS_TEMPLATE | sed
"s/{{MAVEN_PROXY_PORT}}/8888/g"); \
- fi \
- && MAVEN_SETTINGS=$MAVEN_SETTINGS_TEMPLATE \
- && mkdir -p /root/.m2/ \
- && echo $MAVEN_SETTINGS > /root/.m2/settings.xml
-
-# Display environment information
-RUN ulimit -a
-RUN env
-RUN cat /root/.m2/settings.xml
-
-# Install deps from repo
-ARG OS_IMAGE_TAG
-COPY centos-$OS_IMAGE_TAG-deps.sh /tmp/deps.sh
-RUN /tmp/deps.sh \
- && rm /tmp/deps.sh \
- && yum clean all \
- && dnf clean all \
- && rm -rf /var/cache/yum
-
-# Spark binaries
-WORKDIR /opt
-ARG BUILD_SPARK_BINARIES
-
-# Build & install Spark 3.2.2
-RUN if [ "$BUILD_SPARK_BINARIES" = "ON" ]; then wget
https://archive.apache.org/dist/spark/spark-3.2.2/spark-3.2.2-bin-hadoop3.2.tgz;
fi
-RUN if [ "$BUILD_SPARK_BINARIES" = "ON" ]; then mkdir spark322 && tar -xvf
spark-3.2.2-bin-hadoop3.2.tgz -C spark322 --strip-components=1; fi
-
-# Build & install Spark 3.3.1
-RUN if [ "$BUILD_SPARK_BINARIES" = "ON" ]; then wget
https://archive.apache.org/dist/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz;
fi
-RUN if [ "$BUILD_SPARK_BINARIES" = "ON" ]; then mkdir spark331 && tar -xvf
spark-3.3.1-bin-hadoop3.tgz -C spark331 --strip-components=1; fi
-
-# Build & install Spark 3.4.3
-RUN if [ "$BUILD_SPARK_BINARIES" = "ON" ]; then
https://archive.apache.org/dist/spark/spark-3.4.3/spark-3.4.3-bin-hadoop3.tgz;
fi
-RUN if [ "$BUILD_SPARK_BINARIES" = "ON" ]; then mkdir spark343 && tar -xvf
spark-3.4.3-bin-hadoop3.tgz -C spark343 --strip-components=1; fi
-
-# Build & install Spark 3.5.1
-RUN if [ "$BUILD_SPARK_BINARIES" = "ON" ]; then
https://archive.apache.org/dist/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz;
fi
-RUN if [ "$BUILD_SPARK_BINARIES" = "ON" ]; then mkdir spark351 && tar -xvf
spark-3.5.1-bin-hadoop3.tgz -C spark351 --strip-components=1; fi
diff --git
a/tools/gluten-te/centos/examples/buildhere-veloxbe-portable-libs/README.md
b/tools/gluten-te/centos/examples/buildhere-veloxbe-portable-libs/README.md
deleted file mode 100644
index 5574dd0d72..0000000000
--- a/tools/gluten-te/centos/examples/buildhere-veloxbe-portable-libs/README.md
+++ /dev/null
@@ -1,33 +0,0 @@
-# Utility for building C++ libs in CentOS 7 (with glibc 2.17) container
-
-The folder contains script code to build `libvelox.so` and `libgluten.so` in
docker container and for host use.
-
-## Prerequisites
-
-1. X86 CPU architecture
-2. Host machine with Linux operating system
-3. Docker
-
-## Usage
-
-```sh
-# 1. (Optional) Set the following envs in case you are behind http proxy.
-export HTTP_PROXY_HOST=myproxy.example.com
-export HTTP_PROXY_PORT=55555
-
-# 2. Build the C++ libs in a centos 7 docker container.
-# Note, this command could take much longer time to finish if it's never run
before.
-# After the first run, the essential build environment will be cached in
docker builder.
-#
-# Additionally, changes to HTTP_PROXY_HOST / HTTP_PROXY_PORT could invalidate
the build cache
-# either. For more details, please check docker file `dockerfile-buildenv`.
-cd gluten/
-tools/gluten-te/centos/examples/buildhere-veloxbe-portable-libs/run-default.sh
-
-# 3. Check the built libs.
-ls -l cpp/build/releases/
-
-# 4. If you intend to build Gluten's bundled jar, continue running subsequent
Maven commands.
-# For example:
-mvn clean install -P spark-3.4,backends-velox -DskipTests
-```
\ No newline at end of file
diff --git
a/tools/gluten-te/centos/examples/buildhere-veloxbe-portable-libs/run-default.sh
b/tools/gluten-te/centos/examples/buildhere-veloxbe-portable-libs/run-default.sh
deleted file mode 100755
index da217b2eca..0000000000
---
a/tools/gluten-te/centos/examples/buildhere-veloxbe-portable-libs/run-default.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-BASEDIR=$(readlink -f $(dirname $0))
-
-$BASEDIR/run.sh --enable_vcpkg=ON --build_tests=OFF --build_benchmarks=OFF
--enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON
diff --git
a/tools/gluten-te/centos/examples/buildhere-veloxbe-portable-libs/run.sh
b/tools/gluten-te/centos/examples/buildhere-veloxbe-portable-libs/run.sh
deleted file mode 100755
index 36f6b2c74d..0000000000
--- a/tools/gluten-te/centos/examples/buildhere-veloxbe-portable-libs/run.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-BASEDIR=$(readlink -f $(dirname $0))
-
-TIMESTAMP=$(date +%s)
-
-# Set the following env to install Gluten's modified Arrow Jars on host.
-export MOUNT_MAVEN_CACHE=ON
-export EXTRA_DOCKER_OPTIONS="--name buildhere-veloxbe-portable-libs-$TIMESTAMP
-v $BASEDIR/scripts:/opt/scripts"
-
-BASH_ARGS="$*"
-
-$BASEDIR/../../cbash-mount.sh "/opt/scripts/all.sh $BASH_ARGS"
diff --git
a/tools/gluten-te/centos/examples/buildhere-veloxbe-portable-libs/scripts/all.sh
b/tools/gluten-te/centos/examples/buildhere-veloxbe-portable-libs/scripts/all.sh
deleted file mode 100755
index 3b3e69b9e3..0000000000
---
a/tools/gluten-te/centos/examples/buildhere-veloxbe-portable-libs/scripts/all.sh
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-export NUM_THREADS=$(nproc)
-export CMAKE_BUILD_PARALLEL_LEVEL=$(nproc)
-
-# Retry code copied from https://unix.stackexchange.com/a/137639.
-function fail {
- echo $1 >&2
- exit 1
-}
-
-function retry {
- local n=1
- local max=5
- local delay=15
- while true; do
- "$@" && break || {
- if [[ $n -lt $max ]]; then
- ((n++))
- echo "Command failed. Attempt $n/$max:"
- sleep $delay;
- else
- fail "The command has failed after $n attempts."
- fi
- }
- done
-}
-
-# FIXME: Works only in CentOS 7
-source /opt/rh/devtoolset-11/enable
-
-cd /opt/gluten
-
-BASH_ARGS=$@
-
-retry dev/builddeps-veloxbe.sh $BASH_ARGS
diff --git a/tools/gluten-te/centos/gha/gha-checkout/checkout.sh
b/tools/gluten-te/centos/gha/gha-checkout/checkout.sh
deleted file mode 100755
index a1722283b5..0000000000
--- a/tools/gluten-te/centos/gha/gha-checkout/checkout.sh
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-BASEDIR=$(readlink -f $(dirname $0))
-
-source "$BASEDIR/../../defaults.conf"
-
-if [ -z "$GITHUB_RUN_ID" ]
-then
- echo "Unable to parse GITHUB_RUN_ID."
- exit 1
-fi
-
-if [ -z "$GITHUB_JOB" ]
-then
- echo "Unable to parse GITHUB_JOB."
- exit 1
-fi
-
-if [ -z "$GITHUB_SHA" ]
-then
- echo "Unable to parse GITHUB_SHA."
- exit 1
-fi
-
-export EXTRA_DOCKER_OPTIONS="$EXTRA_DOCKER_OPTIONS --name
gha-checkout-$GITHUB_JOB-$GITHUB_RUN_ID --detach -v
$BASEDIR/scripts:/opt/scripts"
-export NON_INTERACTIVE=ON
-
-$BASEDIR/../../cbash-build.sh 'sleep 14400'
-
-# The target branches
-TARGET_GLUTEN_REPO=${TARGET_GLUTEN_REPO:-$DEFAULT_GLUTEN_REPO}
-
-$BASEDIR/exec.sh "/opt/scripts/init.sh $TARGET_GLUTEN_REPO $GITHUB_SHA"
diff --git a/tools/gluten-te/centos/gha/gha-checkout/clean.sh
b/tools/gluten-te/centos/gha/gha-checkout/clean.sh
deleted file mode 100755
index 5514a22a3e..0000000000
--- a/tools/gluten-te/centos/gha/gha-checkout/clean.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-if [ -z "$GITHUB_RUN_ID" ]
-then
- echo "Unable to parse GITHUB_RUN_ID."
- exit 1
-fi
-
-docker stop gha-checkout-$GITHUB_JOB-$GITHUB_RUN_ID || true
diff --git a/tools/gluten-te/centos/gha/gha-checkout/cp.sh
b/tools/gluten-te/centos/gha/gha-checkout/cp.sh
deleted file mode 100755
index 9c0a29cae3..0000000000
--- a/tools/gluten-te/centos/gha/gha-checkout/cp.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-if [ -z "$GITHUB_RUN_ID" ]
-then
- echo "Unable to parse GITHUB_RUN_ID."
- exit 1
-fi
-
-SRC_PATH=$1
-DST_PATH=$2
-
-mkdir -p $DST_PATH
-docker cp gha-checkout-$GITHUB_JOB-$GITHUB_RUN_ID:$1 $2
diff --git a/tools/gluten-te/centos/gha/gha-checkout/exec.sh
b/tools/gluten-te/centos/gha/gha-checkout/exec.sh
deleted file mode 100755
index 7e21504010..0000000000
--- a/tools/gluten-te/centos/gha/gha-checkout/exec.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-CBASH_BASH_ARGS="$*"
-BASH_ARGS="$CBASH_BASH_ARGS"
-
-if [ -z "$GITHUB_RUN_ID" ]
-then
- echo "Unable to parse GITHUB_RUN_ID."
- exit 1
-fi
-
-docker exec gha-checkout-$GITHUB_JOB-$GITHUB_RUN_ID bash -c "cd /opt/gluten &&
$BASH_ARGS"
diff --git a/tools/gluten-te/centos/gha/gha-checkout/scripts/init.sh
b/tools/gluten-te/centos/gha/gha-checkout/scripts/init.sh
deleted file mode 100755
index bddac06f00..0000000000
--- a/tools/gluten-te/centos/gha/gha-checkout/scripts/init.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-TARGET_GLUTEN_REPO=$1
-TARGET_GLUTEN_COMMIT=$2
-
-cd /opt/gluten
-
-git fetch $TARGET_GLUTEN_REPO $TARGET_GLUTEN_COMMIT:build_$TARGET_GLUTEN_COMMIT
-git checkout build_$TARGET_GLUTEN_COMMIT
diff --git a/tools/gluten-te/centos/shared.sh b/tools/gluten-te/centos/shared.sh
deleted file mode 100755
index 962eea4967..0000000000
--- a/tools/gluten-te/centos/shared.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-SHARED_BASEDIR=$(dirname $0)
-
-source "$SHARED_BASEDIR/defaults.conf"
-
-# Enable buildkit
-export DOCKER_BUILDKIT=1
-export BUILDKIT_PROGRESS=plain
-
-# Validate envs
-if [ -z "$HOME" ]
-then
- echo 'Environment variable $HOME not found. Aborting.'
- exit 1
-fi
-
-# Set operating system
-OS_IMAGE_NAME=${OS_IMAGE_NAME:-$DEFAULT_OS_IMAGE_NAME}
-
-# Set os version
-OS_IMAGE_TAG=${OS_IMAGE_TAG:-$DEFAULT_OS_IMAGE_TAG}
-
-# Buildenv will result in this image
-DOCKER_TARGET_IMAGE_BUILDENV=${DOCKER_TARGET_IMAGE_BUILDENV:-$DEFAULT_DOCKER_TARGET_IMAGE_BUILDENV}
-
-# Build will result in this image
-DOCKER_TARGET_IMAGE_BUILD=${DOCKER_TARGET_IMAGE_BUILD:-$DEFAULT_DOCKER_TARGET_IMAGE_BUILD}
-
-DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE="$DOCKER_TARGET_IMAGE_BUILDENV-$OS_IMAGE_NAME:$OS_IMAGE_TAG"
-
-DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE="$DOCKER_TARGET_IMAGE_BUILD-$OS_IMAGE_NAME:$OS_IMAGE_TAG"
diff --git a/tools/gluten-te/ubuntu/README.md b/tools/gluten-te/ubuntu/README.md
deleted file mode 100644
index d293ed1af6..0000000000
--- a/tools/gluten-te/ubuntu/README.md
+++ /dev/null
@@ -1,106 +0,0 @@
-# Portable Test Environment of Gluten (gluten-te)
-
-Build and run [gluten](https://github.com/apache/gluten) and
[gluten-it](https://github.com/apache/gluten/tree/main/tools/gluten-it) in a
portable docker container, from scratch.
-
-# Prerequisites
-
-Only Linux and MacOS are currently supported. Before running the scripts, make
sure you have `git` and `docker` installed in your host machine.
-
-# Getting Started (Build Gluten code, Velox backend)
-
-```sh
-git clone -b main https://github.com/apache/gluten.git gluten # Gluten main
code
-
-export HTTP_PROXY_HOST=myproxy.example.com # in case you are behind http proxy
-export HTTP_PROXY_PORT=55555 # in case you are behind http proxy
-
-cd gluten/
-tools/gluten-te/ubuntu/examples/buildhere-veloxbe/run.sh
-```
-
-# Getting Started (TPC, Velox backend)
-
-```sh
-git clone -b main https://github.com/apache/gluten.git gluten # Gluten main
code
-
-export HTTP_PROXY_HOST=myproxy.example.com # in case you are behind http proxy
-export HTTP_PROXY_PORT=55555 # in case you are behind http proxy
-
-cd gluten/gluten-te
-./tpc.sh
-```
-
-# Configurations
-
-See the [config
file](https://github.com/apache/gluten/blob/main/tools/gluten-te/ubuntu/defaults.conf).
You can modify the file to configure gluten-te, or pass env variables during
running the scripts.
-
-# Example Usages
-
-## Example: Build local Gluten code (Velox backend)
-
-```sh
-cd gluten/
-{PATH_TO_GLUTEN_TE}/examples/buildhere-veloxbe/run.sh
-```
-
-## Example: Build local Gluten code behind a http proxy (Velox backend)
-
-```sh
-export HTTP_PROXY_HOST=myproxy.example.com # in case you are behind http proxy
-export HTTP_PROXY_PORT=55555 # in case you are behind http proxy
-
-cd gluten/
-{PATH_TO_GLUTEN_TE}/examples/buildhere-veloxbe/run.sh
-```
-
-## Example: Run specific maven commands
-
-```sh
-cd gluten/
-{PATH_TO_GLUTEN_TE}/cbash-mount.sh mvn clean dependency:tree
-```
-
-## Example: Run GUI-based IDEs
-
-```sh
-export HTTP_PROXY_HOST=myproxy.example.com # in case you are behind http proxy
-export HTTP_PROXY_PORT=55555 # in case you are behind http proxy
-
-cd gluten/
-{PATH_TO_GLUTEN_TE}/examples/buildhere-veloxbe-dev/run.sh
-
-# In a new host shell, run below command to setup a X11-enabled ssh
-# connection (default password: 123).
-#
-# {DOCKER_CONTAINER_IP} can be found via "docker ps" and "docker inspect",
-# or by checking "ifconfig" results inside the container's shell
-ssh -X root@{DOCKER_CONTAINER_IP}
-```
-
-## Example: Build and run TPC benchmark on non-default remote branches of
Gluten (Velox backend)
-
-```sh
-export HTTP_PROXY_HOST=myproxy.example.com # in case you are behind http proxy
-export HTTP_PROXY_PORT=55555 # in case you are behind http proxy
-
-TARGET_GLUTEN_REPO=my_repo \
-TARGET_GLUTEN_BRANCH=my_branch \
-{PATH_TO_GLUTEN_TE}/tpc.sh
-```
-
-## Example: Build and run TPC benchmark on official latest code behind a http
proxy (Velox backend)
-
-```sh
-export HTTP_PROXY_HOST=myproxy.example.com # in case you are behind http proxy
-export HTTP_PROXY_PORT=55555 # in case you are behind http proxy
-
-{PATH_TO_GLUTEN_TE}/tpc.sh
-```
-
-## Example: Create debug build for all codes, and open a GDB debugger
interface during running gluten-it (Velox backend)
-
-```sh
-DEBUG_BUILD=ON \
-RUN_GDB=ON \
-{PATH_TO_GLUTEN_TE}/tpc.sh
-```
diff --git a/tools/gluten-te/ubuntu/build.sh b/tools/gluten-te/ubuntu/build.sh
deleted file mode 100755
index ddab2be860..0000000000
--- a/tools/gluten-te/ubuntu/build.sh
+++ /dev/null
@@ -1,92 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-BASEDIR=$(dirname $0)
-source "$BASEDIR/buildenv.sh"
-
-## Debug build flags
-
-# Create debug build
-DEBUG_BUILD=${DEBUG_BUILD:-$DEFAULT_DEBUG_BUILD}
-
-if [ -n $JDK_DEBUG_BUILD ]
-then
- echo "Do not set JDK_DEBUG_BUILD manually!"
-fi
-
-if [ -n $GLUTEN_DEBUG_BUILD ]
-then
- echo "Do not set GLUTEN_DEBUG_BUILD manually!"
-fi
-
-if [ "$DEBUG_BUILD" == "ON" ]
-then
- JDK_DEBUG_BUILD=OFF
- GLUTEN_DEBUG_BUILD=ON
-else
- JDK_DEBUG_BUILD=OFF
- GLUTEN_DEBUG_BUILD=OFF
-fi
-
-# The branches used to prepare dependencies
-CACHE_GLUTEN_REPO=${CACHE_GLUTEN_REPO:-$DEFAULT_GLUTEN_REPO}
-CACHE_GLUTEN_BRANCH=${CACHE_GLUTEN_BRANCH:-$DEFAULT_GLUTEN_BRANCH}
-
-CACHE_GLUTEN_COMMIT="$(git ls-remote $CACHE_GLUTEN_REPO $CACHE_GLUTEN_BRANCH |
awk '{print $1;}')"
-
-if [ -z "$CACHE_GLUTEN_COMMIT" ]
-then
- echo "Unable to parse CACHE_GLUTEN_COMMIT."
- exit 1
-fi
-
-# Backend type
-BUILD_BACKEND_TYPE=${BUILD_BACKEND_TYPE:-$DEFAULT_BUILD_BACKEND_TYPE}
-
-##
-
-BUILD_DOCKER_BUILD_ARGS=
-
-BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --ulimit nofile=8192:8192"
-BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --build-arg
BUILDKIT_INLINE_CACHE=1"
-BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --build-arg
DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE=$DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE"
-BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --build-arg
JDK_DEBUG_BUILD=$JDK_DEBUG_BUILD"
-BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --build-arg
GLUTEN_DEBUG_BUILD=$GLUTEN_DEBUG_BUILD"
-BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --build-arg
CACHE_GLUTEN_REPO=$CACHE_GLUTEN_REPO"
-BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --build-arg
CACHE_GLUTEN_COMMIT=$CACHE_GLUTEN_COMMIT"
-BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --build-arg
BUILD_BACKEND_TYPE=$BUILD_BACKEND_TYPE"
-BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS -f $BASEDIR/dockerfile-build"
-BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --target gluten-build"
-BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS -t
$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE"
-
-if [ -n "$DOCKER_CACHE_REGISTRY" ]
-then
- BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --cache-from
$DOCKER_CACHE_REGISTRY/$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE"
-fi
-
-BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS $BASEDIR"
-
-docker build $BUILD_DOCKER_BUILD_ARGS
-
-if [ -n "$DOCKER_PUSH_REGISTRY" ]
-then
- docker tag "$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE"
"$DOCKER_PUSH_REGISTRY/$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE"
- docker push "$DOCKER_PUSH_REGISTRY/$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE"
-fi
-
-# EOF
diff --git a/tools/gluten-te/ubuntu/buildenv.sh
b/tools/gluten-te/ubuntu/buildenv.sh
deleted file mode 100755
index 5888183408..0000000000
--- a/tools/gluten-te/ubuntu/buildenv.sh
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-BASEDIR=$(dirname $0)
-
-source "$BASEDIR/shared.sh"
-
-# Docker registry used to pull layer cache to speed-up builds
-DOCKER_CACHE_REGISTRY=${DOCKER_CACHE_REGISTRY:-$DEFAULT_DOCKER_CACHE_REGISTRY}
-
-# Docker registry to push pre-built images
-DOCKER_PUSH_REGISTRY=${DOCKER_PUSH_REGISTRY:-$DEFAULT_DOCKER_PUSH_REGISTRY}
-
-# HTTP proxy
-HTTP_PROXY_HOST=${HTTP_PROXY_HOST:-$DEFAULT_HTTP_PROXY_HOST}
-HTTP_PROXY_PORT=${HTTP_PROXY_PORT:-$DEFAULT_HTTP_PROXY_PORT}
-
-# If on, use maven mirror settings for PRC's network environment
-USE_ALI_MAVEN_MIRROR=${USE_ALI_MAVEN_MIRROR:-$DEFAULT_USE_ALI_MAVEN_MIRROR}
-
-# Whether to build Spark binaries in buildenv image
-BUILD_SPARK_BINARIES=${BUILD_SPARK_BINARIES:-$DEFAULT_BUILD_SPARK_BINARIES}
-
-# Set timezone name
-TIMEZONE=${TIMEZONE:-$DEFAULT_TIMEZONE}
-
-if [ "$USE_ALI_MAVEN_MIRROR" == "ON" ]
-then
- MAVEN_MIRROR_URL='https://maven.aliyun.com/repository/public'
-else
- MAVEN_MIRROR_URL=
-fi
-
-##
-
-BUILDENV_DOCKER_BUILD_ARGS=
-
-BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --ulimit
nofile=8192:8192"
-BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --build-arg
BUILDKIT_INLINE_CACHE=1"
-BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --build-arg
TIMEZONE=$TIMEZONE"
-BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --build-arg
OS_IMAGE_NAME=$OS_IMAGE_NAME --build-arg OS_IMAGE_TAG=$OS_IMAGE_TAG"
-BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --build-arg
MAVEN_MIRROR_URL=$MAVEN_MIRROR_URL"
-BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --build-arg
HTTP_PROXY_HOST=$HTTP_PROXY_HOST"
-BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --build-arg
HTTP_PROXY_PORT=$HTTP_PROXY_PORT"
-BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --build-arg
BUILD_SPARK_BINARIES=$BUILD_SPARK_BINARIES"
-BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS -f
$BASEDIR/dockerfile-buildenv"
-BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --target
gluten-buildenv"
-BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS -t
$DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE"
-
-if [ -n "$DOCKER_CACHE_REGISTRY" ]
-then
- BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --cache-from
$DOCKER_CACHE_REGISTRY/$DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE"
-fi
-
-BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS $BASEDIR"
-
-docker build $BUILDENV_DOCKER_BUILD_ARGS
-
-if [ -n "$DOCKER_PUSH_REGISTRY" ]
-then
- docker tag "$DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE"
"$DOCKER_PUSH_REGISTRY/$DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE"
- docker push
"$DOCKER_PUSH_REGISTRY/$DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE"
-fi
-
-# EOF
diff --git a/tools/gluten-te/ubuntu/cbash-build.sh
b/tools/gluten-te/ubuntu/cbash-build.sh
deleted file mode 100755
index e874bf37ac..0000000000
--- a/tools/gluten-te/ubuntu/cbash-build.sh
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-BASEDIR=$(dirname $0)
-
-source "$BASEDIR/shared.sh"
-
-# Docker registry used to pull layer cache to speed-up builds
-DOCKER_PULL_REGISTRY=${DOCKER_PULL_REGISTRY:-$DEFAULT_DOCKER_PULL_REGISTRY}
-
-# Non-interactive during docker run
-NON_INTERACTIVE=${NON_INTERACTIVE:-$DEFAULT_NON_INTERACTIVE}
-
-# Do not remove stopped docker container
-PRESERVE_CONTAINER=${PRESERVE_CONTAINER:-$DEFAULT_PRESERVE_CONTAINER}
-
-# Docker options
-EXTRA_DOCKER_OPTIONS=${EXTRA_DOCKER_OPTIONS:-$DEFAULT_EXTRA_DOCKER_OPTIONS}
-
-# Whether to mount Maven cache
-MOUNT_MAVEN_CACHE=${MOUNT_MAVEN_CACHE:-$DEFAULT_MOUNT_MAVEN_CACHE}
-
-if [ -n "$DOCKER_PULL_REGISTRY" ]
-then
- docker pull $DOCKER_PULL_REGISTRY/$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE
- docker tag $DOCKER_PULL_REGISTRY/$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE
$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE
-else
- $BASEDIR/build.sh
-fi
-
-CBASH_DOCKER_RUN_ARGS=
-if [ "$NON_INTERACTIVE" != "ON" ]
-then
- CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS -it"
-fi
-if [ "$PRESERVE_CONTAINER" != "ON" ]
-then
- CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --rm"
-fi
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --init"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --privileged"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --ulimit nofile=65536:65536"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --ulimit core=-1"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --security-opt
seccomp=unconfined"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS -e NUM_THREADS=$(nproc)"
-if [ "$MOUNT_MAVEN_CACHE" == "ON" ]
-then
- CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS -v
$HOME/.m2/repository:/root/.m2/repository"
-fi
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS -v $HOME/.ccache:/root/.ccache"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS $EXTRA_DOCKER_OPTIONS"
-
-CBASH_BASH_ARGS="$*"
-BASH_ARGS="$CBASH_BASH_ARGS"
-
-docker run $CBASH_DOCKER_RUN_ARGS $DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE "cd
/opt/gluten && $BASH_ARGS"
diff --git a/tools/gluten-te/ubuntu/cbash-mount.sh
b/tools/gluten-te/ubuntu/cbash-mount.sh
deleted file mode 100755
index 4432ef33db..0000000000
--- a/tools/gluten-te/ubuntu/cbash-mount.sh
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-BASEDIR=$(dirname $0)
-
-source "$BASEDIR/buildenv.sh"
-
-# Non-interactive during docker run
-NON_INTERACTIVE=${NON_INTERACTIVE:-$DEFAULT_NON_INTERACTIVE}
-
-# Do not remove stopped docker container
-PRESERVE_CONTAINER=${PRESERVE_CONTAINER:-$DEFAULT_PRESERVE_CONTAINER}
-
-# Docker options
-EXTRA_DOCKER_OPTIONS=${EXTRA_DOCKER_OPTIONS:-$DEFAULT_EXTRA_DOCKER_OPTIONS}
-
-# Whether to mount Maven cache
-MOUNT_MAVEN_CACHE=${MOUNT_MAVEN_CACHE:-$DEFAULT_MOUNT_MAVEN_CACHE}
-
-CBASH_DOCKER_RUN_ARGS=
-if [ "$NON_INTERACTIVE" != "ON" ]
-then
- CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS -it"
-fi
-if [ "$PRESERVE_CONTAINER" != "ON" ]
-then
- CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --rm"
-fi
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --init"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --privileged"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --ulimit nofile=65536:65536"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --ulimit core=-1"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --security-opt
seccomp=unconfined"
-if [ "$MOUNT_MAVEN_CACHE" == "ON" ]
-then
- CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS -v
$HOME/.m2/repository:/root/.m2/repository"
-fi
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS -v $HOME/.ccache:/root/.ccache"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS -v $PWD:/opt/gluten"
-CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS $EXTRA_DOCKER_OPTIONS"
-
-CBASH_BASH_ARGS="$*"
-BASH_ARGS="$CBASH_BASH_ARGS"
-
-docker run $CBASH_DOCKER_RUN_ARGS $DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE
"cd /opt/gluten && $BASH_ARGS"
diff --git a/tools/gluten-te/ubuntu/defaults.conf
b/tools/gluten-te/ubuntu/defaults.conf
deleted file mode 100644
index c4f3c76497..0000000000
--- a/tools/gluten-te/ubuntu/defaults.conf
+++ /dev/null
@@ -1,94 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-## For basic scripts
-
-# Non-interactive during docker run
-DEFAULT_NON_INTERACTIVE=OFF
-
-# Do not remove stopped docker container
-DEFAULT_PRESERVE_CONTAINER=OFF
-
-# The codes will be used in build
-DEFAULT_GLUTEN_REPO=https://github.com/apache/gluten.git
-DEFAULT_GLUTEN_BRANCH=main
-
-# Create debug build
-DEFAULT_DEBUG_BUILD=OFF
-
-# Backend type (velox)
-DEFAULT_BUILD_BACKEND_TYPE=velox
-
-# HTTP proxy
-# If http proxy is http://XXX.com:123, write it as
-# DEFAULT_HTTP_PROXY_HOST=XXX.com
-# DEFAULT_HTTP_PROXY_PORT=123
-# do not write http://
-DEFAULT_HTTP_PROXY_HOST=
-DEFAULT_HTTP_PROXY_PORT=
-
-# If on, use maven mirror settings for PRC's network environment
-DEFAULT_USE_ALI_MAVEN_MIRROR=OFF
-
-# Whether to build Spark binaries in buildenv image
-DEFAULT_BUILD_SPARK_BINARIES=OFF
-
-# Base operator system image used in build scripts.
-DEFAULT_OS_IMAGE_NAME=ubuntu
-
-# Version ID of os image
-DEFAULT_OS_IMAGE_TAG=20.04
-
-# Set timezone name
-DEFAULT_TIMEZONE=Asia/Shanghai
-
-# Build will result in this image
-DEFAULT_DOCKER_TARGET_IMAGE_BUILDENV=gluten-te/gluten-buildenv
-
-# Build will result in this image
-DEFAULT_DOCKER_TARGET_IMAGE_BUILD=gluten-te/gluten-build
-
-# Docker registry to push pre-built images
-DEFAULT_DOCKER_PULL_REGISTRY=
-
-# Docker registry used to pull layer cache to speed-up builds
-DEFAULT_DOCKER_CACHE_REGISTRY=
-
-# Docker registry to push pre-built images
-DEFAULT_DOCKER_PUSH_REGISTRY=
-
-## For tpc.sh
-
-# Java options
-DEFAULT_EXTRA_JAVA_OPTIONS="-Xmx2G"
-
-# Run GDB.
-DEFAULT_RUN_GDB=OFF
-
-# Run GDB server.
-DEFAULT_RUN_GDB_SERVER=OFF
-
-# GDB server bind port
-DEFAULT_GDB_SERVER_PORT=2345
-
-# Run JVM jdwp server.
-DEFAULT_RUN_JDWP_SERVER=OFF
-
-# JVM jdwp bind port
-DEFAULT_JDWP_SERVER_PORT=5005
-
-# Docker options
-DEFAULT_EXTRA_DOCKER_OPTIONS="--network bridge"
-
-# Build will result in this image
-DEFAULT_DOCKER_TARGET_IMAGE_TPC=gluten-te/gluten-tpc
-DEFAULT_DOCKER_TARGET_IMAGE_TPC_GDB=gluten-te/gluten-tpc-gdb
-DEFAULT_DOCKER_TARGET_IMAGE_TPC_GDB_SERVER=gluten-te/gluten-tpc-gdb-server
-
-## For cbash.sh
-
-# Whether to mount Maven cache
-DEFAULT_MOUNT_MAVEN_CACHE=OFF
-
-# EOF
diff --git a/tools/gluten-te/ubuntu/docker-compose.yml
b/tools/gluten-te/ubuntu/docker-compose.yml
deleted file mode 100644
index 6e14581182..0000000000
--- a/tools/gluten-te/ubuntu/docker-compose.yml
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-TODO
-
diff --git a/tools/gluten-te/ubuntu/dockerfile-build
b/tools/gluten-te/ubuntu/dockerfile-build
deleted file mode 100644
index 7f2c883465..0000000000
--- a/tools/gluten-te/ubuntu/dockerfile-build
+++ /dev/null
@@ -1,106 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-ARG DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE
-
-FROM $DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE AS gluten-build
-MAINTAINER Hongze Zhang<[email protected]>
-
-# Whether debug build is enabled
-ARG JDK_DEBUG_BUILD
-ARG GLUTEN_DEBUG_BUILD
-RUN echo "JDK debug build is [$JDK_DEBUG_BUILD]!"
-RUN echo "Gluten debug build is [$GLUTEN_DEBUG_BUILD]!"
-
-# If JDK debug is on
-RUN if [ "$JDK_DEBUG_BUILD" == "ON" ]; \
- then \
- apt-get update; \
- DEBIAN_FRONTEND=noninteractive apt-get uninstall -y openjdk-8-jdk; \
- DEBIAN_FRONTEND=noninteractive apt-get uninstall -y maven; \
- mkdir -p /opt/jdk/ \
- && mkdir -p /opt/maven/ \
- && cd /opt/jdk/ \
- && wget
https://builds.shipilev.net/openjdk-jdk8/openjdk-jdk8-linux-x86_64-server-fastdebug-gcc8-glibc2.28.tar.xz
\
- && tar -xvf
openjdk-jdk8-linux-x86_64-server-fastdebug-gcc8-glibc2.28.tar.xz \
- && rm -f
openjdk-jdk8-linux-x86_64-server-fastdebug-gcc8-glibc2.28.tar.xz \
- && cd /opt/maven/ \
- && wget
https://dlcdn.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz
\
- && tar -xvf apache-maven-3.6.3-bin.tar.gz \
- && rm -f apache-maven-3.6.3-bin.tar.gz \
- && cp -rs /opt/jdk/j2sdk-image/bin/* /usr/local/bin/ \
- && cp -rs /opt/maven/apache-maven-3.6.3/bin/mvn /usr/local/bin/ \
- && echo "JAVA_HOME=/opt/jdk/j2sdk-image" > ~/.mavenrc; \
- fi
-
-# These branches are mainly for pre-downloading dependencies to speed-up
builds.
-# Thus it should not be required to change these values every time when the
build branch
-# is changed.
-ARG CACHE_GLUTEN_REPO
-ARG CACHE_GLUTEN_COMMIT
-
-RUN test -n "$CACHE_GLUTEN_REPO" || (echo "CACHE_GLUTEN_REPO not set" && false)
-RUN test -n "$CACHE_GLUTEN_COMMIT" || (echo "CACHE_GLUTEN_COMMIT not set" &&
false)
-
-RUN cd /opt \
- && git clone $CACHE_GLUTEN_REPO gluten \
- && cd gluten \
- && git fetch $CACHE_GLUTEN_REPO
$CACHE_GLUTEN_COMMIT:build_$CACHE_GLUTEN_COMMIT \
- && git checkout build_$CACHE_GLUTEN_COMMIT
-
-# Set ccache size
-RUN ccache -M 128G
-RUN ccache -s
-
-# Default Gluten Maven build options (empty as of now)
-ENV GLUTEN_MAVEN_OPTIONS=
-RUN set-login-env "GLUTEN_MAVEN_OPTIONS="
-
-ARG BUILD_BACKEND_TYPE
-
-RUN test -n "$BUILD_BACKEND_TYPE" || (echo "BUILD_BACKEND_TYPE not set" &&
false)
-
-RUN if [ "$BUILD_BACKEND_TYPE" == "velox" ]; \
- then \
- if [ "$GLUTEN_DEBUG_BUILD" == "ON" ]; then GLUTEN_BUILD_TYPE="Debug";
else GLUTEN_BUILD_TYPE="Release"; fi; \
- DEPS_INSTALL_SCRIPT="bash /opt/gluten/dev/builddeps-veloxbe.sh \
- --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON
--enable_abfs=ON \
- --build_type=$GLUTEN_BUILD_TYPE"; \
- EXTRA_MAVEN_OPTIONS="-Pspark-3.5 \
- -Pbackends-velox \
- -Pceleborn \
- -Puniffle \
- -Piceberg \
- -Pdelta \
- -DskipTests \
- -Dscalastyle.skip=true \
- -Dcheckstyle.skip=true"; \
- else \
- echo "Unrecognizable backend type: $BUILD_BACKEND_TYPE"; \
- exit 1; \
- fi \
- && echo $EXTRA_MAVEN_OPTIONS > ~/.gluten-mvn-options \
- && echo $DEPS_INSTALL_SCRIPT > ~/.gluten-deps-install-script
-
-# Prebuild Gluten
-RUN EXTRA_MAVEN_OPTIONS=$(cat ~/.gluten-mvn-options) \
- DEPS_INSTALL_SCRIPT=$(cat ~/.gluten-deps-install-script) \
- && cd /opt/gluten \
- && bash -c "$DEPS_INSTALL_SCRIPT" \
- && bash -c "mvn clean install $GLUTEN_MAVEN_OPTIONS $EXTRA_MAVEN_OPTIONS"
-
-# EOF
diff --git a/tools/gluten-te/ubuntu/dockerfile-buildenv
b/tools/gluten-te/ubuntu/dockerfile-buildenv
deleted file mode 100644
index 29596e20ae..0000000000
--- a/tools/gluten-te/ubuntu/dockerfile-buildenv
+++ /dev/null
@@ -1,162 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-ARG OS_IMAGE_NAME
-ARG OS_IMAGE_TAG
-
-FROM $OS_IMAGE_NAME:$OS_IMAGE_TAG AS gluten-buildenv
-MAINTAINER Hongze Zhang<[email protected]>
-
-SHELL ["/bin/bash", "-l", "-c"]
-ENTRYPOINT ["/bin/bash", "-l", "-c"]
-CMD ["/bin/bash"]
-
-# Add script for adding environment variables for login-shell (e.g. a shell
via ssh)
-COPY scripts/set-login-env.sh /usr/local/sbin/set-login-env
-
-# REQUIRED PROXIES: APT, WGET, GIT, MAVEN (also Maven mirror)
-ARG HTTP_PROXY_HOST
-ARG HTTP_PROXY_PORT
-
-# Sometimes ENV a=b won't work when the shell is not docker-default, so we
-# use both two ways to set the variables
-ENV http_proxy=${HTTP_PROXY_HOST:+"http://$HTTP_PROXY_HOST:$HTTP_PROXY_PORT"}
-ENV https_proxy=${HTTP_PROXY_HOST:+"http://$HTTP_PROXY_HOST:$HTTP_PROXY_PORT"}
-ENV no_proxy=localhost,127.0.0.1,127.0.0.0/8,172.16.0.0/12,192.168.0.0/16
-ENV HTTP_PROXY=${HTTP_PROXY_HOST:+"http://$HTTP_PROXY_HOST:$HTTP_PROXY_PORT"}
-ENV HTTPS_PROXY=${HTTP_PROXY_HOST:+"http://$HTTP_PROXY_HOST:$HTTP_PROXY_PORT"}
-ENV NO_PROXY=localhost,127.0.0.1,127.0.0.0/8,172.16.0.0/12,192.168.0.0/16
-RUN set-login-env
"http_proxy=${HTTP_PROXY_HOST:+http://$HTTP_PROXY_HOST:$HTTP_PROXY_PORT}";
-RUN set-login-env
"https_proxy=${HTTP_PROXY_HOST:+http://$HTTP_PROXY_HOST:$HTTP_PROXY_PORT}";
-RUN set-login-env
"no_proxy=localhost,127.0.0.1,127.0.0.0/8,172.16.0.0/12,192.168.0.0/16"
-RUN set-login-env
"HTTP_PROXY=${HTTP_PROXY_HOST:+http://$HTTP_PROXY_HOST:$HTTP_PROXY_PORT}";
-RUN set-login-env
"HTTPS_PROXY=${HTTP_PROXY_HOST:+http://$HTTP_PROXY_HOST:$HTTP_PROXY_PORT}";
-RUN set-login-env
"NO_PROXY=localhost,127.0.0.1,127.0.0.0/8,172.16.0.0/12,192.168.0.0/16"
-
-RUN if [ -n "$HTTP_PROXY_HOST" ]; then echo "Acquire::http::Proxy
\"http://$HTTP_PROXY_HOST:$HTTP_PROXY_PORT\";"; >> /etc/apt/apt.conf; fi
-RUN if [ -n "$HTTP_PROXY_HOST" ]; then echo "Acquire::https::Proxy
\"http://$HTTP_PROXY_HOST:$HTTP_PROXY_PORT\";"; >> /etc/apt/apt.conf; fi
-
-ARG MAVEN_MIRROR_URL
-
-RUN if [ -n "$MAVEN_MIRROR_URL" ]; \
- then \
-
MAVEN_SETTINGS_TEMPLATE="<settings><mirrors><mirror><id>mavenmirror</id><mirrorOf>central</mirrorOf><name>MavenMirror</name><url>{{MAVEN_MIRROR_URL}}</url></mirror></mirrors><proxies><proxy><id>httpproxy</id><active>{{MAVEN_PROXY_ENABLE}}</active><protocol>http</protocol><host>{{MAVEN_PROXY_HOST}}</host><port>{{MAVEN_PROXY_PORT}}</port></proxy><proxy><id>httpsproxy</id><active>{{MAVEN_PROXY_ENABLE}}</active><protocol>https</protocol><host>{{MAVEN_PROXY_HOST}}</host><port>{{MAVEN_PR
[...]
- MAVEN_SETTINGS_TEMPLATE=$(echo $MAVEN_SETTINGS_TEMPLATE | sed
"s@{{MAVEN_MIRROR_URL}}@$MAVEN_MIRROR_URL@g"); \
- else \
-
MAVEN_SETTINGS_TEMPLATE="<settings><proxies><proxy><id>httpproxy</id><active>{{MAVEN_PROXY_ENABLE}}</active><protocol>http</protocol><host>{{MAVEN_PROXY_HOST}}</host><port>{{MAVEN_PROXY_PORT}}</port></proxy><proxy><id>httpsproxy</id><active>{{MAVEN_PROXY_ENABLE}}</active><protocol>https</protocol><host>{{MAVEN_PROXY_HOST}}</host><port>{{MAVEN_PROXY_PORT}}</port></proxy></proxies></settings>";
\
- fi \
- && if [ -n "$HTTP_PROXY_HOST" ]; \
- then \
- MAVEN_SETTINGS_TEMPLATE=$(echo $MAVEN_SETTINGS_TEMPLATE | sed
"s/{{MAVEN_PROXY_ENABLE}}/true/g"); \
- MAVEN_SETTINGS_TEMPLATE=$(echo $MAVEN_SETTINGS_TEMPLATE | sed
"s/{{MAVEN_PROXY_HOST}}/$HTTP_PROXY_HOST/g"); \
- MAVEN_SETTINGS_TEMPLATE=$(echo $MAVEN_SETTINGS_TEMPLATE | sed
"s/{{MAVEN_PROXY_PORT}}/$HTTP_PROXY_PORT/g"); \
- else \
- MAVEN_SETTINGS_TEMPLATE=$(echo $MAVEN_SETTINGS_TEMPLATE | sed
"s/{{MAVEN_PROXY_ENABLE}}/false/g"); \
- MAVEN_SETTINGS_TEMPLATE=$(echo $MAVEN_SETTINGS_TEMPLATE | sed
"s/{{MAVEN_PROXY_HOST}}/localhost/g"); \
- MAVEN_SETTINGS_TEMPLATE=$(echo $MAVEN_SETTINGS_TEMPLATE | sed
"s/{{MAVEN_PROXY_PORT}}/8888/g"); \
- fi \
- && MAVEN_SETTINGS=$MAVEN_SETTINGS_TEMPLATE \
- && mkdir -p /root/.m2/ \
- && echo $MAVEN_SETTINGS > /root/.m2/settings.xml
-
-# Display environment information
-RUN ulimit -a
-RUN env
-RUN cat /etc/apt/apt.conf || (echo "Apt proxy not set" && true)
-RUN cat /root/.m2/settings.xml
-
-## APT dependencies
-
-# Update, then install essentials
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y sudo
locales wget tar tzdata git ccache ninja-build build-essential llvm-11-dev
clang-11 libiberty-dev libdwarf-dev libre2-dev libz-dev libssl-dev
libboost-all-dev libcurl4-openssl-dev curl zip unzip tar pkg-config
autoconf-archive bison flex
-
-# install HBM dependencies
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y
autoconf automake g++ libnuma-dev libtool numactl unzip libdaxctl-dev
-
-# Install OpenJDK 8 and Maven
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y
openjdk-8-jdk
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y maven
-
-# Setup SSH server
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y
openssh-server
-RUN systemctl disable ssh
-RUN ssh-keygen -A
-RUN mkdir -p /run/sshd
-RUN echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config.d/override.conf
-RUN echo 'X11Forwarding yes' >> /etc/ssh/sshd_config.d/override.conf
-RUN echo 'X11UseLocalhost no' >> /etc/ssh/sshd_config.d/override.conf
-RUN echo -e "123\n123" | passwd
-
-ARG TIMEZONE
-RUN test -n "$TIMEZONE" || (echo "TIMEZONE not set" && false)
-
-RUN TZ=$TIMEZONE \
- && ln -snf /usr/share/zoneinfo/$TZ /etc/localtime \
- && echo $TZ > /etc/timezone \
- && dpkg-reconfigure -f noninteractive tzdata
-
-# Configure locale
-RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen \
- && locale-gen
-
-ENV LANG=en_US.UTF-8
-ENV LANGUAGE=en_US:en
-ENV LC_ALL=en_US.UTF-8
-RUN set-login-env "LANG=en_US.UTF-8"
-RUN set-login-env "LANGUAGE=en_US:en"
-RUN set-login-env "LC_ALL=en_US.UTF-8"
-
-# Install CMake
-RUN cd /opt && wget
https://github.com/Kitware/CMake/releases/download/v3.28.3/cmake-3.28.3-linux-x86_64.sh
\
- && mkdir cmake \
- && bash cmake-3.28.3-linux-x86_64.sh --skip-license --prefix=/opt/cmake \
- && ln -s /opt/cmake/bin/cmake /usr/bin/cmake
-RUN cmake --version
-
-# Install GCC 11
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y
software-properties-common
-RUN add-apt-repository ppa:ubuntu-toolchain-r/test
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-11
g++-11
-RUN rm -f /usr/bin/gcc /usr/bin/g++
-RUN ln -s /usr/bin/gcc-11 /usr/bin/gcc
-RUN ln -s /usr/bin/g++-11 /usr/bin/g++
-RUN cc --version
-RUN c++ --version
-
-# Spark binaries
-WORKDIR /opt
-ARG BUILD_SPARK_BINARIES
-
-# Build & install Spark 3.2.2
-RUN if [ "$BUILD_SPARK_BINARIES" = "ON" ]; then wget
https://archive.apache.org/dist/spark/spark-3.2.2/spark-3.2.2-bin-hadoop3.2.tgz;
fi
-RUN if [ "$BUILD_SPARK_BINARIES" = "ON" ]; then mkdir spark322 && tar -xvf
spark-3.2.2-bin-hadoop3.2.tgz -C spark322 --strip-components=1; fi
-
-# Build & install Spark 3.3.1
-RUN if [ "$BUILD_SPARK_BINARIES" = "ON" ]; then wget
https://archive.apache.org/dist/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz;
fi
-RUN if [ "$BUILD_SPARK_BINARIES" = "ON" ]; then mkdir spark331 && tar -xvf
spark-3.3.1-bin-hadoop3.tgz -C spark331 --strip-components=1; fi
-
-# Build & install Spark 3.4.3
-RUN if [ "$BUILD_SPARK_BINARIES" = "ON" ]; then
https://archive.apache.org/dist/spark/spark-3.4.3/spark-3.4.3-bin-hadoop3.tgz;
fi
-RUN if [ "$BUILD_SPARK_BINARIES" = "ON" ]; then mkdir spark343 && tar -xvf
spark-3.4.3-bin-hadoop3.tgz -C spark343 --strip-components=1; fi
-
-# Build & install Spark 3.5.1
-RUN if [ "$BUILD_SPARK_BINARIES" = "ON" ]; then
https://archive.apache.org/dist/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz;
fi
-RUN if [ "$BUILD_SPARK_BINARIES" = "ON" ]; then mkdir spark351 && tar -xvf
spark-3.5.1-bin-hadoop3.tgz -C spark351 --strip-components=1; fi
-
-# Prepare entry command
-COPY scripts/cmd.sh /root/.cmd.sh
-CMD ["/root/.cmd.sh"]
diff --git a/tools/gluten-te/ubuntu/dockerfile-tpc
b/tools/gluten-te/ubuntu/dockerfile-tpc
deleted file mode 100644
index eac2814584..0000000000
--- a/tools/gluten-te/ubuntu/dockerfile-tpc
+++ /dev/null
@@ -1,62 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-ARG DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE
-
-FROM $DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE AS gluten-tpc
-MAINTAINER Hongze Zhang<[email protected]>
-
-# Incrementally build Gluten against target commit based on previous build
-ARG TARGET_GLUTEN_REPO
-ARG TARGET_GLUTEN_COMMIT
-
-RUN test -n "$TARGET_GLUTEN_REPO" || (echo "TARGET_GLUTEN_REPO not set" &&
false)
-RUN test -n "$TARGET_GLUTEN_COMMIT" || (echo "TARGET_GLUTEN_COMMIT not set" &&
false)
-
-RUN cd /opt/gluten \
- && git fetch $TARGET_GLUTEN_REPO
$TARGET_GLUTEN_COMMIT:build_$TARGET_GLUTEN_COMMIT \
- && git checkout build_$TARGET_GLUTEN_COMMIT
-
-RUN EXTRA_MAVEN_OPTIONS=$(cat ~/.gluten-mvn-options) \
- DEPS_INSTALL_SCRIPT=$(cat ~/.gluten-deps-install-script) \
- && cd /opt/gluten \
- && bash -c "$DEPS_INSTALL_SCRIPT" \
- && bash -c "mvn clean install $GLUTEN_MAVEN_OPTIONS $EXTRA_MAVEN_OPTIONS" \
- && bash -c "ln -sf /opt/gluten/ep/build-velox/build/velox_ep /opt/velox"
-
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y
--no-install-recommends ssh net-tools vim unzip libjemalloc-dev
-
-ARG BUILD_BACKEND_TYPE
-
-RUN test -n "$BUILD_BACKEND_TYPE" || (echo "BUILD_BACKEND_TYPE not set" &&
false)
-
-RUN GLUTEN_IT_MAVEN_OPTIONS="-Pspark-3.2" \
- && echo $GLUTEN_IT_MAVEN_OPTIONS > ~/.gluten-it-mvn-options;
-
-RUN GLUTEN_IT_MAVEN_OPTIONS=$(cat ~/.gluten-it-mvn-options) \
- && cd /opt/gluten/tools/gluten-it \
- && bash -c "mvn clean install $GLUTEN_IT_MAVEN_OPTIONS"
-
-FROM gluten-tpc AS gluten-tpc-gdb
-
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y
--no-install-recommends gdb
-
-FROM gluten-tpc-gdb AS gluten-tpc-gdb-server
-
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y
--no-install-recommends gdbserver
-
-# EOF
diff --git a/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-dev/run.sh
b/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-dev/run.sh
deleted file mode 100755
index 2baa2900e1..0000000000
--- a/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-dev/run.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-BASEDIR=$(readlink -f $(dirname $0))
-
-TIMESTAMP=$(date +%s)
-
-export EXTRA_DOCKER_OPTIONS="--name buildhere-veloxbe-dev-$TIMESTAMP --detach
-v $BASEDIR/scripts:/opt/scripts"
-export PRESERVE_CONTAINER=ON
-
-$BASEDIR/../../cbash-mount.sh 'bash /root/.cmd.sh'
-docker exec buildhere-veloxbe-dev-$TIMESTAMP '/opt/scripts/all.sh'
diff --git
a/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-dev/scripts/all.sh
b/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-dev/scripts/all.sh
deleted file mode 100755
index d69e3d4b60..0000000000
--- a/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-dev/scripts/all.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-# Build Gluten
-EXTRA_MAVEN_OPTIONS="-Pspark-3.2 \
- -Pbackends-velox \
- -Pceleborn \
- -Puniffle \
- -DskipTests \
- -Dscalastyle.skip=true \
- -Dcheckstyle.skip=true"
-
-cd /opt/gluten
-bash dev/builddeps-veloxbe.sh --build_type=Debug
-mvn clean install $EXTRA_MAVEN_OPTIONS
-
-apt-get -y -q --no-install-recommends install firefox tmux openjdk-8-source
-
-# Install IDEs
-apt-get update
-apt-get -y -q --no-install-recommends install libgbm-dev libxkbcommon-dev
-mkdir -p /opt/ide
-cd /opt/ide
-wget https://download.jetbrains.com/idea/ideaIC-2022.3.2.tar.gz
-tar -xvzf ideaIC-2022.3.2.tar.gz
-wget https://download.jetbrains.com/cpp/CLion-2022.3.2.tar.gz
-tar -xvzf CLion-2022.3.2.tar.gz
diff --git
a/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-portable-libs/README.md
b/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-portable-libs/README.md
deleted file mode 100644
index 8660265e35..0000000000
--- a/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-portable-libs/README.md
+++ /dev/null
@@ -1,33 +0,0 @@
-# Utility for building C++ libs in Ubuntu 20.04 (with glibc 2.31) container
-
-The folder contains script code to build `libvelox.so` and `libgluten.so` in
docker container and for host use.
-
-## Prerequisites
-
-1. X86 CPU architecture
-2. Host machine with Linux operating system
-3. Docker
-
-## Usage
-
-```sh
-# 1. (Optional) Set the following envs in case you are behind http proxy.
-export HTTP_PROXY_HOST=myproxy.example.com
-export HTTP_PROXY_PORT=55555
-
-# 2. Build the C++ libs in a ubuntu 20.04 docker container.
-# Note, this command could take much longer time to finish if it's never run
before.
-# After the first run, the essential build environment will be cached in
docker builder.
-#
-# Additionally, changes to HTTP_PROXY_HOST / HTTP_PROXY_PORT could invalidate
the build cache
-# either. For more details, please check docker file `dockerfile-buildenv`.
-cd gluten/
-tools/gluten-te/ubuntu/examples/buildhere-veloxbe-portable-libs/run-default.sh
-
-# 3. Check the built libs.
-ls -l cpp/build/releases/
-
-# 4. If you intend to build Gluten's bundled jar, continue running subsequent
Maven commands.
-# For example:
-mvn clean install -P spark-3.4,backends-velox -DskipTests
-```
\ No newline at end of file
diff --git
a/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-portable-libs/run-default.sh
b/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-portable-libs/run-default.sh
deleted file mode 100755
index da217b2eca..0000000000
---
a/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-portable-libs/run-default.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-BASEDIR=$(readlink -f $(dirname $0))
-
-$BASEDIR/run.sh --enable_vcpkg=ON --build_tests=OFF --build_benchmarks=OFF
--enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON
diff --git
a/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-portable-libs/run.sh
b/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-portable-libs/run.sh
deleted file mode 100755
index 36f6b2c74d..0000000000
--- a/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-portable-libs/run.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-BASEDIR=$(readlink -f $(dirname $0))
-
-TIMESTAMP=$(date +%s)
-
-# Set the following env to install Gluten's modified Arrow Jars on host.
-export MOUNT_MAVEN_CACHE=ON
-export EXTRA_DOCKER_OPTIONS="--name buildhere-veloxbe-portable-libs-$TIMESTAMP
-v $BASEDIR/scripts:/opt/scripts"
-
-BASH_ARGS="$*"
-
-$BASEDIR/../../cbash-mount.sh "/opt/scripts/all.sh $BASH_ARGS"
diff --git
a/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-portable-libs/scripts/all.sh
b/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-portable-libs/scripts/all.sh
deleted file mode 100755
index c9db960eb3..0000000000
---
a/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-portable-libs/scripts/all.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-export NUM_THREADS=$(nproc)
-export CMAKE_BUILD_PARALLEL_LEVEL=$(nproc)
-
-# Retry code copied from https://unix.stackexchange.com/a/137639.
-function fail {
- echo $1 >&2
- exit 1
-}
-
-function retry {
- local n=1
- local max=5
- local delay=15
- while true; do
- "$@" && break || {
- if [[ $n -lt $max ]]; then
- ((n++))
- echo "Command failed. Attempt $n/$max:"
- sleep $delay;
- else
- fail "The command has failed after $n attempts."
- fi
- }
- done
-}
-
-cd /opt/gluten
-retry apt-get update
-retry apt-get install -y --dry-run # We now have all essentials installed in
image.
-
-BASH_ARGS=$@
-
-retry dev/builddeps-veloxbe.sh $BASH_ARGS
diff --git a/tools/gluten-te/ubuntu/examples/buildhere-veloxbe/run.sh
b/tools/gluten-te/ubuntu/examples/buildhere-veloxbe/run.sh
deleted file mode 100755
index a8bbeea83d..0000000000
--- a/tools/gluten-te/ubuntu/examples/buildhere-veloxbe/run.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-BASEDIR=$(dirname $0)
-
-EXTRA_MAVEN_OPTIONS="-Pspark-3.2 \
- -Pbackends-velox \
- -Pceleborn \
- -Puniffle \
- -DskipTests \
- -Dscalastyle.skip=true \
- -Dcheckstyle.skip=true"
-
-$BASEDIR/../../cbash-mount.sh "cd /opt/gluten && dev/builddeps-veloxbe.sh &&
mvn clean install $EXTRA_MAVEN_OPTIONS"
diff --git a/tools/gluten-te/ubuntu/gha/gha-checkout/checkout.sh
b/tools/gluten-te/ubuntu/gha/gha-checkout/checkout.sh
deleted file mode 100755
index a1722283b5..0000000000
--- a/tools/gluten-te/ubuntu/gha/gha-checkout/checkout.sh
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-BASEDIR=$(readlink -f $(dirname $0))
-
-source "$BASEDIR/../../defaults.conf"
-
-if [ -z "$GITHUB_RUN_ID" ]
-then
- echo "Unable to parse GITHUB_RUN_ID."
- exit 1
-fi
-
-if [ -z "$GITHUB_JOB" ]
-then
- echo "Unable to parse GITHUB_JOB."
- exit 1
-fi
-
-if [ -z "$GITHUB_SHA" ]
-then
- echo "Unable to parse GITHUB_SHA."
- exit 1
-fi
-
-export EXTRA_DOCKER_OPTIONS="$EXTRA_DOCKER_OPTIONS --name
gha-checkout-$GITHUB_JOB-$GITHUB_RUN_ID --detach -v
$BASEDIR/scripts:/opt/scripts"
-export NON_INTERACTIVE=ON
-
-$BASEDIR/../../cbash-build.sh 'sleep 14400'
-
-# The target branches
-TARGET_GLUTEN_REPO=${TARGET_GLUTEN_REPO:-$DEFAULT_GLUTEN_REPO}
-
-$BASEDIR/exec.sh "/opt/scripts/init.sh $TARGET_GLUTEN_REPO $GITHUB_SHA"
diff --git a/tools/gluten-te/ubuntu/gha/gha-checkout/clean.sh
b/tools/gluten-te/ubuntu/gha/gha-checkout/clean.sh
deleted file mode 100755
index 5514a22a3e..0000000000
--- a/tools/gluten-te/ubuntu/gha/gha-checkout/clean.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-if [ -z "$GITHUB_RUN_ID" ]
-then
- echo "Unable to parse GITHUB_RUN_ID."
- exit 1
-fi
-
-docker stop gha-checkout-$GITHUB_JOB-$GITHUB_RUN_ID || true
diff --git a/tools/gluten-te/ubuntu/gha/gha-checkout/cp.sh
b/tools/gluten-te/ubuntu/gha/gha-checkout/cp.sh
deleted file mode 100755
index 9c0a29cae3..0000000000
--- a/tools/gluten-te/ubuntu/gha/gha-checkout/cp.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-if [ -z "$GITHUB_RUN_ID" ]
-then
- echo "Unable to parse GITHUB_RUN_ID."
- exit 1
-fi
-
-SRC_PATH=$1
-DST_PATH=$2
-
-mkdir -p $DST_PATH
-docker cp gha-checkout-$GITHUB_JOB-$GITHUB_RUN_ID:$1 $2
diff --git a/tools/gluten-te/ubuntu/gha/gha-checkout/exec.sh
b/tools/gluten-te/ubuntu/gha/gha-checkout/exec.sh
deleted file mode 100755
index 7e21504010..0000000000
--- a/tools/gluten-te/ubuntu/gha/gha-checkout/exec.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-CBASH_BASH_ARGS="$*"
-BASH_ARGS="$CBASH_BASH_ARGS"
-
-if [ -z "$GITHUB_RUN_ID" ]
-then
- echo "Unable to parse GITHUB_RUN_ID."
- exit 1
-fi
-
-docker exec gha-checkout-$GITHUB_JOB-$GITHUB_RUN_ID bash -c "cd /opt/gluten &&
$BASH_ARGS"
diff --git a/tools/gluten-te/ubuntu/gha/gha-checkout/scripts/init.sh
b/tools/gluten-te/ubuntu/gha/gha-checkout/scripts/init.sh
deleted file mode 100755
index bddac06f00..0000000000
--- a/tools/gluten-te/ubuntu/gha/gha-checkout/scripts/init.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-TARGET_GLUTEN_REPO=$1
-TARGET_GLUTEN_COMMIT=$2
-
-cd /opt/gluten
-
-git fetch $TARGET_GLUTEN_REPO $TARGET_GLUTEN_COMMIT:build_$TARGET_GLUTEN_COMMIT
-git checkout build_$TARGET_GLUTEN_COMMIT
diff --git a/tools/gluten-te/ubuntu/scripts/cmd.sh
b/tools/gluten-te/ubuntu/scripts/cmd.sh
deleted file mode 100755
index 787c4ac620..0000000000
--- a/tools/gluten-te/ubuntu/scripts/cmd.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-echo "Running gluten-buildenv docker container with SSH X11 forwarding
enabled. SSH password: 123"
-/usr/sbin/sshd -D
diff --git a/tools/gluten-te/ubuntu/scripts/set-login-env.sh
b/tools/gluten-te/ubuntu/scripts/set-login-env.sh
deleted file mode 100755
index bce3f89162..0000000000
--- a/tools/gluten-te/ubuntu/scripts/set-login-env.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-ENV="$*"
-echo "export $ENV" >> /etc/profile.d/99-gluten.sh
diff --git a/tools/gluten-te/ubuntu/shared.sh b/tools/gluten-te/ubuntu/shared.sh
deleted file mode 100755
index 962eea4967..0000000000
--- a/tools/gluten-te/ubuntu/shared.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-SHARED_BASEDIR=$(dirname $0)
-
-source "$SHARED_BASEDIR/defaults.conf"
-
-# Enable buildkit
-export DOCKER_BUILDKIT=1
-export BUILDKIT_PROGRESS=plain
-
-# Validate envs
-if [ -z "$HOME" ]
-then
- echo 'Environment variable $HOME not found. Aborting.'
- exit 1
-fi
-
-# Set operating system
-OS_IMAGE_NAME=${OS_IMAGE_NAME:-$DEFAULT_OS_IMAGE_NAME}
-
-# Set os version
-OS_IMAGE_TAG=${OS_IMAGE_TAG:-$DEFAULT_OS_IMAGE_TAG}
-
-# Buildenv will result in this image
-DOCKER_TARGET_IMAGE_BUILDENV=${DOCKER_TARGET_IMAGE_BUILDENV:-$DEFAULT_DOCKER_TARGET_IMAGE_BUILDENV}
-
-# Build will result in this image
-DOCKER_TARGET_IMAGE_BUILD=${DOCKER_TARGET_IMAGE_BUILD:-$DEFAULT_DOCKER_TARGET_IMAGE_BUILD}
-
-DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE="$DOCKER_TARGET_IMAGE_BUILDENV-$OS_IMAGE_NAME:$OS_IMAGE_TAG"
-
-DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE="$DOCKER_TARGET_IMAGE_BUILD-$OS_IMAGE_NAME:$OS_IMAGE_TAG"
diff --git a/tools/gluten-te/ubuntu/tpc.sh b/tools/gluten-te/ubuntu/tpc.sh
deleted file mode 100755
index eaa44d2854..0000000000
--- a/tools/gluten-te/ubuntu/tpc.sh
+++ /dev/null
@@ -1,154 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-BASEDIR=$(dirname $0)
-
-source "$BASEDIR/build.sh"
-
-# Non-interactive during docker run
-NON_INTERACTIVE=${NON_INTERACTIVE:-$DEFAULT_NON_INTERACTIVE}
-
-# Do not remove stopped docker container
-PRESERVE_CONTAINER=${PRESERVE_CONTAINER:-$DEFAULT_PRESERVE_CONTAINER}
-
-# Java options
-EXTRA_JAVA_OPTIONS=${EXTRA_JAVA_OPTIONS:-$DEFAULT_EXTRA_JAVA_OPTIONS}
-
-# Docker options
-EXTRA_DOCKER_OPTIONS=${EXTRA_DOCKER_OPTIONS:-$DEFAULT_EXTRA_DOCKER_OPTIONS}
-
-# Run GDB.
-RUN_GDB=${RUN_GDB:-$DEFAULT_RUN_GDB}
-
-# Run GDB server.
-RUN_GDB_SERVER=${RUN_GDB_SERVER:-$DEFAULT_RUN_GDB_SERVER}
-
-# Run JVM jdwp server.
-RUN_JDWP_SERVER=${RUN_JDWP_SERVER:-$DEFAULT_RUN_JDWP_SERVER}
-
-if [ "$RUN_GDB" == "ON" ] && [ "$RUN_GDB_SERVER" == "ON" ]
-then
- echo "RUN_GDB_SERVER and RUN_GDB_SERVER can not be turned on at the same
time."
- exit 1
-fi
-
-
-if [ "$RUN_GDB" == "ON" ]
-then
-
DOCKER_SELECTED_TARGET_IMAGE_TPC=${DOCKER_TARGET_IMAGE_TPC_GDB:-$DEFAULT_DOCKER_TARGET_IMAGE_TPC_GDB}
- DOCKER_BUILD_TARGET_NAME=gluten-tpc-gdb
-elif [ "$RUN_GDB_SERVER" == "ON" ]
-then
-
DOCKER_SELECTED_TARGET_IMAGE_TPC=${DOCKER_TARGET_IMAGE_TPC_GDB_SERVER:-$DEFAULT_DOCKER_TARGET_IMAGE_TPC_GDB_SERVER}
- DOCKER_BUILD_TARGET_NAME=gluten-tpc-gdb-server
-else
-
DOCKER_SELECTED_TARGET_IMAGE_TPC=${DOCKER_TARGET_IMAGE_TPC:-$DEFAULT_DOCKER_TARGET_IMAGE_TPC}
- DOCKER_BUILD_TARGET_NAME=gluten-tpc
-fi
-
-# The target branches
-TARGET_GLUTEN_REPO=${TARGET_GLUTEN_REPO:-$DEFAULT_GLUTEN_REPO}
-TARGET_GLUTEN_BRANCH=${TARGET_GLUTEN_BRANCH:-$DEFAULT_GLUTEN_BRANCH}
-
-## Fetch target commit
-
-TARGET_GLUTEN_COMMIT="$(git ls-remote $TARGET_GLUTEN_REPO
$TARGET_GLUTEN_BRANCH | awk '{print $1;}')"
-
-if [ -z "$TARGET_GLUTEN_COMMIT" ]
-then
- echo "Unable to parse TARGET_GLUTEN_COMMIT."
- exit 1
-fi
-
-DOCKER_SELECTED_TARGET_IMAGE_TPC_WITH_OS_IMAGE="$DOCKER_SELECTED_TARGET_IMAGE_TPC-$OS_IMAGE:$OS_IMAGE_TAG"
-
-TPC_DOCKER_BUILD_ARGS=
-TPC_DOCKER_BUILD_ARGS="$TPC_DOCKER_BUILD_ARGS --ulimit nofile=8192:8192"
-TPC_DOCKER_BUILD_ARGS="$TPC_DOCKER_BUILD_ARGS --build-arg
BUILDKIT_INLINE_CACHE=1"
-TPC_DOCKER_BUILD_ARGS="$TPC_DOCKER_BUILD_ARGS --build-arg
DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE=$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE"
-TPC_DOCKER_BUILD_ARGS="$TPC_DOCKER_BUILD_ARGS --build-arg
TARGET_GLUTEN_REPO=$TARGET_GLUTEN_REPO"
-TPC_DOCKER_BUILD_ARGS="$TPC_DOCKER_BUILD_ARGS --build-arg
TARGET_GLUTEN_COMMIT=$TARGET_GLUTEN_COMMIT"
-TPC_DOCKER_BUILD_ARGS="$TPC_DOCKER_BUILD_ARGS --build-arg
BUILD_BACKEND_TYPE=$BUILD_BACKEND_TYPE"
-TPC_DOCKER_BUILD_ARGS="$TPC_DOCKER_BUILD_ARGS -f $BASEDIR/dockerfile-tpc"
-TPC_DOCKER_BUILD_ARGS="$TPC_DOCKER_BUILD_ARGS --target
$DOCKER_BUILD_TARGET_NAME"
-TPC_DOCKER_BUILD_ARGS="$TPC_DOCKER_BUILD_ARGS -t
$DOCKER_SELECTED_TARGET_IMAGE_TPC_WITH_OS_IMAGE"
-
-# GDB server bind port
-GDB_SERVER_PORT=${GDB_SERVER_PORT:-$DEFAULT_GDB_SERVER_PORT}
-
-# JVM jdwp bind port
-JDWP_SERVER_PORT=${JDWP_SERVER_PORT:-$DEFAULT_JDWP_SERVER_PORT}
-
-if [ -n "$DOCKER_CACHE_REGISTRY" ]
-then
- docker pull
$DOCKER_CACHE_REGISTRY/$DOCKER_SELECTED_TARGET_IMAGE_TPC_WITH_OS_IMAGE || echo
"Required image not found in cache registry."
- TPC_DOCKER_BUILD_ARGS="$TPC_DOCKER_BUILD_ARGS --cache-from
$DOCKER_CACHE_REGISTRY/$DOCKER_SELECTED_TARGET_IMAGE_TPC_WITH_OS_IMAGE"
-fi
-
-TPC_DOCKER_BUILD_ARGS="$TPC_DOCKER_BUILD_ARGS $BASEDIR"
-
-docker build $TPC_DOCKER_BUILD_ARGS
-
-if [ -n "$DOCKER_PUSH_REGISTRY" ]
-then
- docker tag "$DOCKER_SELECTED_TARGET_IMAGE_TPC_WITH_OS_IMAGE"
"$DOCKER_PUSH_REGISTRY/$DOCKER_SELECTED_TARGET_IMAGE_TPC_WITH_OS_IMAGE"
- docker push
"$DOCKER_PUSH_REGISTRY/$DOCKER_SELECTED_TARGET_IMAGE_TPC_WITH_OS_IMAGE"
-fi
-
-TPC_DOCKER_RUN_ARGS=
-if [ "$NON_INTERACTIVE" != "ON" ]
-then
- TPC_DOCKER_RUN_ARGS="$TPC_DOCKER_RUN_ARGS -it"
-fi
-if [ "$PRESERVE_CONTAINER" != "ON" ]
-then
- CBASH_DOCKER_RUN_ARGS="$CBASH_DOCKER_RUN_ARGS --rm"
-fi
-TPC_DOCKER_RUN_ARGS="$TPC_DOCKER_RUN_ARGS --init"
-TPC_DOCKER_RUN_ARGS="$TPC_DOCKER_RUN_ARGS --privileged"
-TPC_DOCKER_RUN_ARGS="$TPC_DOCKER_RUN_ARGS --ulimit nofile=65536:65536"
-TPC_DOCKER_RUN_ARGS="$TPC_DOCKER_RUN_ARGS --ulimit core=-1"
-TPC_DOCKER_RUN_ARGS="$TPC_DOCKER_RUN_ARGS --security-opt seccomp=unconfined"
-TPC_DOCKER_RUN_ARGS="$TPC_DOCKER_RUN_ARGS $EXTRA_DOCKER_OPTIONS"
-
-TPC_CMD_ARGS="$*"
-
-JAVA_ARGS=
-if [ "$RUN_JDWP_SERVER" == "ON" ]
-then
- JAVA_ARGS="$JAVA_ARGS -ea"
- JAVA_ARGS="$JAVA_ARGS
-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$JDWP_SERVER_PORT"
-fi
-JAVA_ARGS="$JAVA_ARGS $EXTRA_JAVA_OPTIONS"
-JAVA_ARGS="$JAVA_ARGS -cp '/opt/gluten/tools/gluten-it/package/target/lib/*'"
-JAVA_ARGS="$JAVA_ARGS org.apache.gluten.integration.tpc.Tpc $TPC_CMD_ARGS"
-
-BASH_ARGS=
-if [ "$RUN_GDB" == "ON" ]
-then
- BASH_ARGS="gdb --args java $JAVA_ARGS"
-elif [ "$RUN_GDB_SERVER" == "ON" ]
-then
- BASH_ARGS="$BASH_ARGS gdbserver :$GDB_SERVER_PORT java $JAVA_ARGS"
-else
- BASH_ARGS="java $JAVA_ARGS"
-fi
-
-docker run $TPC_DOCKER_RUN_ARGS
"$DOCKER_SELECTED_TARGET_IMAGE_TPC_WITH_OS_IMAGE" "$BASH_ARGS"
-
-# EOF
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
