This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new aafbc72e28 [deps](libhdfs) change hadoop libhdfs to source compile
(#20077)
aafbc72e28 is described below
commit aafbc72e28fa9dfcc449d7746fd4900c651f74e3
Author: Mingyu Chen <[email protected]>
AuthorDate: Fri May 26 14:34:25 2023 +0800
[deps](libhdfs) change hadoop libhdfs to source compile (#20077)
cherry-pick #20058
---
thirdparty/CHANGELOG.md | 4 ++++
thirdparty/build-thirdparty.sh | 19 +++++++++++--------
thirdparty/vars.sh | 10 +++++-----
3 files changed, 20 insertions(+), 13 deletions(-)
diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md
index e92bd2ade5..6ae976b145 100644
--- a/thirdparty/CHANGELOG.md
+++ b/thirdparty/CHANGELOG.md
@@ -2,6 +2,10 @@
This file contains version of the third-party dependency libraries in the
build-env image. The docker build-env image is apache/doris, and the tag is
`build-env-${version}`
+## v20230525
+
+- Modified: hadoop libhdfs 3.3.4.1
+
## v20230218
- Modified: libhdfs3 2.3.7 -> 2.3.8
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index 9a4babac78..58b629654a 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -1565,14 +1565,17 @@ build_fast_float() {
cp -r ./include/fast_float "${TP_INSTALL_DIR}/include/"
}
-# hadoop_libs_x86
-build_hadoop_libs_x86() {
- check_if_source_exist "${HADOOP_LIBS_X86_SOURCE}"
- cd "${TP_SOURCE_DIR}/${HADOOP_LIBS_X86_SOURCE}"
+# hadoop_libs
+build_hadoop_libs() {
+ check_if_source_exist "${HADOOP_LIBS_SOURCE}"
+ cd "${TP_SOURCE_DIR}/${HADOOP_LIBS_SOURCE}"
+ echo "THIRDPARTY_INSTALLED=${TP_INSTALL_DIR}" >env.sh
+ ./build.sh
+
mkdir -p "${TP_INSTALL_DIR}/include/hadoop_hdfs/"
mkdir -p "${TP_INSTALL_DIR}/lib/hadoop_hdfs/"
- cp ./include/hdfs.h "${TP_INSTALL_DIR}/include/hadoop_hdfs/"
- cp -r ./* "${TP_INSTALL_DIR}/lib/hadoop_hdfs/"
+ cp -r ./hadoop-dist/target/hadoop-libhdfs-3.3.4/*
"${TP_INSTALL_DIR}/lib/hadoop_hdfs/"
+ cp -r ./hadoop-dist/target/hadoop-libhdfs-3.3.4/include/hdfs.h
"${TP_INSTALL_DIR}/include/hadoop_hdfs/"
}
if [[ "${#packages[@]}" -eq 0 ]]; then
@@ -1637,8 +1640,8 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
if [[ "$(uname -s)" == 'Darwin' ]]; then
read -r -a packages <<<"binutils gettext ${packages[*]}"
- elif [[ "$(uname -s)" == 'Linux' ]] && [[ "$(uname -m)" == 'x86_64' ]];
then
- read -r -a packages <<<"${packages[*]} hadoop_libs_x86"
+ elif [[ "$(uname -s)" == 'Linux' ]]; then
+ read -r -a packages <<<"${packages[*]} hadoop_libs"
fi
fi
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 3f7035de47..b37b36e769 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -442,10 +442,10 @@ FAST_FLOAT_SOURCE=fast_float-3.9.0
FAST_FLOAT_MD5SUM="5656b0d8b150a3b157cfb092d214f6ea"
# libhdfs
-HADOOP_LIBS_X86_DOWNLOAD="https://github.com/apache/doris-thirdparty/releases/download/hadoop-3.3.4.1-for-doris/hadoop_lib_3.3.4.1-x86.tar.gz";
-HADOOP_LIBS_X86_NAME="hadoop_lib_3.3.4.1-x86.tar.gz"
-HADOOP_LIBS_X86_SOURCE="hadoop_lib_3.3.4.1-x86"
-HADOOP_LIBS_X86_MD5SUM="9b434ae90f17a358c55b5476841cb836"
+HADOOP_LIBS_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/hadoop-3.3.4.2-for-doris.tar.gz";
+HADOOP_LIBS_NAME="hadoop-3.3.4.2-for-doris.tar.gz"
+HADOOP_LIBS_SOURCE="doris-thirdparty-hadoop-3.3.4.2-for-doris"
+HADOOP_LIBS_MD5SUM="88a09261810a4e3a18d4101600234202"
# all thirdparties which need to be downloaded is set in array TP_ARCHIVES
export TP_ARCHIVES=(
@@ -511,7 +511,7 @@ export TP_ARCHIVES=(
'XXHASH'
'CONCURRENTQUEUE'
'FAST_FLOAT'
- 'HADOOP_LIBS_X86'
+ 'HADOOP_LIBS'
)
if [[ "$(uname -s)" == 'Darwin' ]]; then
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
