This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new eb40d0af75e [Opt](external-docker) Modify kerberos network mode to
host (#47043)
eb40d0af75e is described below
commit eb40d0af75efc8c1d7808d5dd5dd9850ef359e40
Author: zgxme <[email protected]>
AuthorDate: Thu Jan 16 17:59:37 2025 +0800
[Opt](external-docker) Modify kerberos network mode to host (#47043)
### What problem does this PR solve?
In order to be able to perform kerberos testing on a multi-node doris
cluster.
---
.../update-location.sh => common/event-hook.sh} | 14 ++---
.../docker-compose/common/hive-configure.sh | 50 ++++++++++++++++
.../kerberos/common/conf/doris-krb5.conf | 19 +++---
.../kerberos/common/hadoop/hadoop-run.sh | 2 +
.../kerberos/conf/kerberos1/kdc.conf.tpl | 50 ++++++++++++++++
.../kerberos1/krb5.conf.tpl} | 27 +++------
.../kerberos2/kdc.conf.tpl} | 23 ++++++--
.../health.sh => conf/kerberos2/krb5.conf.tpl} | 30 +++++-----
.../hadoop-health-check.sh => conf/my.cnf.tpl} | 36 ++++++-----
.../kerberos/entrypoint-hive-master-2.sh | 50 ----------------
.../kerberos/entrypoint-hive-master.sh | 46 +++++++++++----
.../docker-compose/kerberos/hadoop-hive.env.tpl | 65 ++++++++++++++++++++
.../kerberos/health-checks/health.sh | 1 -
.../kerberos/health-checks/hive-health-check-2.sh | 2 +-
.../kerberos/health-checks/hive-health-check.sh | 2 +-
...doop-health-check.sh => supervisorctl-check.sh} | 0
.../docker-compose/kerberos/kerberos.yaml.tpl | 69 +++++++++-------------
.../docker-compose/kerberos/kerberos1_settings.env | 47 +++++++++++++++
...ntial_cache_files.sh => kerberos2_settings.env} | 40 ++++++++-----
docker/thirdparties/run-thirdparties-docker.sh | 24 +++++---
.../kerberos/test_single_hive_kerberos.groovy | 19 +++---
.../kerberos/test_two_hive_kerberos.groovy | 13 ++--
22 files changed, 411 insertions(+), 218 deletions(-)
diff --git
a/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/update-location.sh
b/docker/thirdparties/docker-compose/common/event-hook.sh
old mode 100755
new mode 100644
similarity index 74%
copy from
docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/update-location.sh
copy to docker/thirdparties/docker-compose/common/event-hook.sh
index 8d727b2308d..144550e8bb0
---
a/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/update-location.sh
+++ b/docker/thirdparties/docker-compose/common/event-hook.sh
@@ -16,10 +16,10 @@
# specific language governing permissions and limitations
# under the License.
-/usr/bin/mysqld_safe &
-while ! mysqladmin ping -proot --silent; do sleep 1; done
-
-hive --service metatool -updateLocation
hdfs://hadoop-master-2:9000/user/hive/warehouse
hdfs://hadoop-master:9000/user/hive/warehouse
-
-killall mysqld
-while pgrep mysqld; do sleep 1; done
+function exec_success_hook() {
+ echo "Executing success hook"
+ echo "Creating /tmp/success and /tmp/SUCCESS"
+ touch /tmp/success /tmp/SUCCESS
+ echo "Do not exit, just tailing /dev/null"
+ tail -f /dev/null
+}
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/common/hive-configure.sh
b/docker/thirdparties/docker-compose/common/hive-configure.sh
new file mode 100755
index 00000000000..cb17d4d2275
--- /dev/null
+++ b/docker/thirdparties/docker-compose/common/hive-configure.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Referenced from [docker-hive](https://github.com/big-data-europe/docker-hive)
+
+# Set some sensible defaults
+export CORE_CONF_fs_defaultFS=${CORE_CONF_fs_defaultFS:-hdfs://`hostname
-f`:8020}
+
+function addProperty() {
+ local path=$1
+ local name=$2
+ local value=$3
+
+ local entry="<property><name>$name</name><value>${value}</value></property>"
+ local escapedEntry=$(echo $entry | sed 's/\//\\\//g')
+ sed -i "/<\/configuration>/ s/.*/${escapedEntry}\n&/" $path
+}
+
+function configure() {
+ local path=$1
+ local module=$2
+ local envPrefix=$3
+
+ local var
+ local value
+
+ echo "Configuring $module"
+ for c in `printenv | perl -sne 'print "$1 " if m/^${envPrefix}_(.+?)=.*/'
-- -envPrefix=$envPrefix`; do
+ name=`echo ${c} | perl -pe 's/___/-/g; s/__/_/g; s/_/./g'`
+ var="${envPrefix}_${c}"
+ value=${!var}
+ echo " - Setting $name=$ "
+ addProperty $path $name "$value"
+ done
+}
\ No newline at end of file
diff --git
a/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf
b/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf
index 36547b8f89d..83fe29c2cb2 100644
--- a/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf
+++ b/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf
@@ -32,21 +32,18 @@
[realms]
LABS.TERADATA.COM = {
- kdc = hadoop-master:88
- admin_server = hadoop-master
+ kdc = hadoop-master:5588
+ admin_server = hadoop-master:5749
}
OTHERLABS.TERADATA.COM = {
- kdc = hadoop-master:89
- admin_server = hadoop-master
+ kdc = hadoop-master:5589
+ admin_server = hadoop-master:5750
}
- OTHERLABS.TERADATA.COM = {
- kdc = hadoop-master:89
- admin_server = hadoop-master
- }
-OTHERREALM.COM = {
- kdc = hadoop-master-2:88
- admin_server = hadoop-master
+ OTHERREALM.COM = {
+ kdc = hadoop-master-2:6688
+ admin_server = hadoop-master-2:6749
}
[domain_realm]
hadoop-master-2 = OTHERREALM.COM
+ hadoop-master = LABS.TERADATA.COM
diff --git
a/docker/thirdparties/docker-compose/kerberos/common/hadoop/hadoop-run.sh
b/docker/thirdparties/docker-compose/kerberos/common/hadoop/hadoop-run.sh
index b8bfd8715e9..93c6e385eff 100755
--- a/docker/thirdparties/docker-compose/kerberos/common/hadoop/hadoop-run.sh
+++ b/docker/thirdparties/docker-compose/kerberos/common/hadoop/hadoop-run.sh
@@ -38,5 +38,7 @@ fi
trap exit INT
echo "Running services with supervisord"
+rm -rf /etc/supervisord.d/socks-proxy.conf
+rm -rf /etc/supervisord.d/sshd.conf
supervisord -c /etc/supervisord.conf
diff --git
a/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/kdc.conf.tpl
b/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/kdc.conf.tpl
new file mode 100644
index 00000000000..e16c70e16db
--- /dev/null
+++ b/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/kdc.conf.tpl
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[kdcdefaults]
+ kdc_ports = ${KDC_PORT1}
+ kdc_tcp_ports = ${KDC_PORT1}
+ kadmind_port = ${KADMIND_PORT1}
+ kpasswd_port = ${KPASSWD_PORT1}
+
+[realms]
+ LABS.TERADATA.COM = {
+ acl_file = /var/kerberos/krb5kdc/kadm5.acl
+ dict_file = /usr/share/dict/words
+ admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab
+ supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal
arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal
+ kdc_listen = ${KDC_PORT1}
+ kdc_tcp_listen = ${KDC_PORT1}
+ kdc_ports = ${KDC_PORT1}
+ kdc_tcp_ports = ${KDC_PORT1}
+ kadmind_port = ${KADMIND_PORT1}
+ kpasswd_port = ${KPASSWD_PORT1}
+ }
+
+ OTHERLABS.TERADATA.COM = {
+ acl_file = /var/kerberos/krb5kdc/kadm5-other.acl
+ dict_file = /usr/share/dict/words
+ admin_keytab = /var/kerberos/krb5kdc/kadm5-other.keytab
+ supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal
arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal
+ kdc_listen = ${KDC_PORT2}
+ kdc_tcp_listen = ${KDC_PORT2}
+ kdc_ports = ${KDC_PORT2}
+ kdc_tcp_ports = ${KDC_PORT2}
+ kadmind_port = ${KADMIND_PORT2}
+ kpasswd_port = ${KPASSWD_PORT2}
+ }
\ No newline at end of file
diff --git
a/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf
b/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/krb5.conf.tpl
similarity index 66%
copy from
docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf
copy to docker/thirdparties/docker-compose/kerberos/conf/kerberos1/krb5.conf.tpl
index 36547b8f89d..1edf2bb8fd0 100644
--- a/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf
+++ b/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/krb5.conf.tpl
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -24,29 +25,15 @@
default_realm = LABS.TERADATA.COM
dns_lookup_realm = false
dns_lookup_kdc = false
- ticket_lifetime = 5s
- # this setting is causing a Message stream modified (41) error when talking
to KDC running on CentOS 7: https://stackoverflow.com/a/60978520
- # renew_lifetime = 7d
forwardable = true
- udp_preference_limit = 1
+ allow_weak_crypto = true
[realms]
LABS.TERADATA.COM = {
- kdc = hadoop-master:88
- admin_server = hadoop-master
+ kdc = ${HOST}:${KDC_PORT1}
+ admin_server = ${HOST}:${KADMIND_PORT1}
}
OTHERLABS.TERADATA.COM = {
- kdc = hadoop-master:89
- admin_server = hadoop-master
- }
- OTHERLABS.TERADATA.COM = {
- kdc = hadoop-master:89
- admin_server = hadoop-master
- }
-OTHERREALM.COM = {
- kdc = hadoop-master-2:88
- admin_server = hadoop-master
- }
-
-[domain_realm]
- hadoop-master-2 = OTHERREALM.COM
+ kdc = ${HOST}:${KDC_PORT2}
+ admin_server = ${HOST}:${KADMIND_PORT2}
+ }
\ No newline at end of file
diff --git
a/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/update-location.sh
b/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/kdc.conf.tpl
old mode 100755
new mode 100644
similarity index 56%
rename from
docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/update-location.sh
rename to
docker/thirdparties/docker-compose/kerberos/conf/kerberos2/kdc.conf.tpl
index 8d727b2308d..61b4994ad5c
---
a/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/update-location.sh
+++ b/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/kdc.conf.tpl
@@ -16,10 +16,23 @@
# specific language governing permissions and limitations
# under the License.
-/usr/bin/mysqld_safe &
-while ! mysqladmin ping -proot --silent; do sleep 1; done
+[kdcdefaults]
+ kdc_ports = ${KDC_PORT1}
+ kdc_tcp_ports = ${KDC_PORT1}
+ kadmind_port = ${KADMIND_PORT1}
+ kpasswd_port = ${KPASSWD_PORT1}
-hive --service metatool -updateLocation
hdfs://hadoop-master-2:9000/user/hive/warehouse
hdfs://hadoop-master:9000/user/hive/warehouse
-killall mysqld
-while pgrep mysqld; do sleep 1; done
+[realms]
+ OTHERREALM.COM = {
+ acl_file = /var/kerberos/krb5kdc/kadm5.acl
+ dict_file = /usr/share/dict/words
+ admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab
+ supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal
arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal
+ kdc_listen = ${KDC_PORT1}
+ kdc_tcp_listen = ${KDC_PORT1}
+ kdc_ports = ${KDC_PORT1}
+ kdc_tcp_ports = ${KDC_PORT1}
+ kadmind_port = ${KADMIND_PORT1}
+ kpasswd_port = ${KPASSWD_PORT1}
+ }
\ No newline at end of file
diff --git
a/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh
b/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/krb5.conf.tpl
old mode 100755
new mode 100644
similarity index 68%
copy from docker/thirdparties/docker-compose/kerberos/health-checks/health.sh
copy to docker/thirdparties/docker-compose/kerberos/conf/kerberos2/krb5.conf.tpl
index 473d7ceaeb6..c817dbdd797
--- a/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh
+++ b/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/krb5.conf.tpl
@@ -16,20 +16,20 @@
# specific language governing permissions and limitations
# under the License.
-set -euo pipefail
+[logging]
+ default = FILE:/var/log/krb5libs.log
+ kdc = FILE:/var/log/krb5kdc.log
+ admin_server = FILE:/var/log/kadmind.log
-if test $# -gt 0; then
- echo "$0 does not accept arguments" >&2
- exit 32
-fi
+[libdefaults]
+ default_realm = OTHERREALM.COM
+ dns_lookup_realm = false
+ dns_lookup_kdc = false
+ forwardable = true
+ allow_weak_crypto = true
-set -x
-
-HEALTH_D=${HEALTH_D:-/etc/health.d/}
-
-if test -d "${HEALTH_D}"; then
- for health_script in "${HEALTH_D}"/*; do
- "${health_script}" &>> /var/log/container-health.log || exit 1
- done
-fi
-exit 0
+[realms]
+ OTHERREALM.COM = {
+ kdc = ${HOST}:${KDC_PORT1}
+ admin_server = ${HOST}:${KADMIND_PORT1}
+ }
\ No newline at end of file
diff --git
a/docker/thirdparties/docker-compose/kerberos/health-checks/hadoop-health-check.sh
b/docker/thirdparties/docker-compose/kerberos/conf/my.cnf.tpl
old mode 100755
new mode 100644
similarity index 57%
copy from
docker/thirdparties/docker-compose/kerberos/health-checks/hadoop-health-check.sh
copy to docker/thirdparties/docker-compose/kerberos/conf/my.cnf.tpl
index 77df431d85a..e91c65c1004
---
a/docker/thirdparties/docker-compose/kerberos/health-checks/hadoop-health-check.sh
+++ b/docker/thirdparties/docker-compose/kerberos/conf/my.cnf.tpl
@@ -16,25 +16,23 @@
# specific language governing permissions and limitations
# under the License.
-set -euo pipefail
+[mysqld]
+port=${MYSQL_PORT}
+datadir=/var/lib/mysql
+socket=/var/lib/mysql/mysql.sock
+# Disabling symbolic-links is recommended to prevent assorted security risks
+symbolic-links=0
+# Settings user and group are ignored when systemd is used.
+# If you need to run mysqld under a different user or group,
+# customize your systemd unit file for mariadb according to the
+# instructions in http://fedoraproject.org/wiki/Systemd
-if test $# -gt 0; then
- echo "$0 does not accept arguments" >&2
- exit 32
-fi
+[mysqld_safe]
+log-error=/var/log/mariadb/mariadb.log
+pid-file=/var/run/mariadb/mariadb.pid
-# Supervisord is not running
-if ! test -f /tmp/supervisor.sock; then
- exit 0
-fi
-
-# Check if all Hadoop services are running
-FAILED=$(supervisorctl status | grep -v RUNNING || true)
+#
+# include all files from the config directory
+#
+!includedir /etc/my.cnf.d
-if [ "$FAILED" == "" ]; then
- echo "All services are running"
- exit 0
-else
- echo "Some of the services are failing: ${FAILED}"
- exit 1
-fi
diff --git
a/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master-2.sh
b/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master-2.sh
deleted file mode 100755
index eb95c5cb697..00000000000
--- a/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master-2.sh
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -euo pipefail
-
-echo "Copying kerberos keytabs to /keytabs/"
-mkdir -p /etc/hadoop-init.d/
-cp /etc/trino/conf/hive-presto-master.keytab
/keytabs/other-hive-presto-master.keytab
-cp /etc/trino/conf/presto-server.keytab /keytabs/other-presto-server.keytab
-cp /keytabs/update-location.sh /etc/hadoop-init.d/update-location.sh
-/usr/local/hadoop-run.sh &
-
-# check healthy hear
-echo "Waiting for hadoop to be healthy"
-
-for i in {1..10}; do
- if /usr/local/health.sh; then
- echo "Hadoop is healthy"
- break
- fi
- echo "Hadoop is not healthy yet. Retrying in 20 seconds..."
- sleep 20
-done
-
-if [ $i -eq 10 ]; then
- echo "Hadoop did not become healthy after 120 attempts. Exiting."
- exit 1
-fi
-
-echo "Init kerberos test data"
-kinit -kt /etc/hive/conf/hive.keytab hive/[email protected]
-hive -f /usr/local/sql/create_kerberos_hive_table.sql
-touch /mnt/SUCCESS
-
-tail -f /dev/null
diff --git
a/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh
b/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh
index 76f49724297..f12b76734f7 100755
--- a/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh
+++ b/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh
@@ -15,34 +15,60 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
set -euo pipefail
+source /usr/local/common/hive-configure.sh
+source /usr/local/common/event-hook.sh
+
+echo "Configuring hive"
+configure /etc/hive/conf/hive-site.xml hive HIVE_SITE_CONF
+configure /etc/hive/conf/hiveserver2-site.xml hive HIVE_SITE_CONF
+configure /etc/hadoop/conf/core-site.xml core CORE_CONF
+configure /etc/hadoop/conf/hdfs-site.xml hdfs HDFS_CONF
+configure /etc/hadoop/conf/yarn-site.xml yarn YARN_CONF
+configure /etc/hadoop/conf/mapred-site.xml mapred MAPRED_CONF
+configure /etc/hive/conf/beeline-site.xml beeline BEELINE_SITE_CONF
echo "Copying kerberos keytabs to keytabs/"
mkdir -p /etc/hadoop-init.d/
-cp /etc/trino/conf/* /keytabs/
+
+if [ "$1" == "1" ]; then
+ cp /etc/trino/conf/* /keytabs/
+elif [ "$1" == "2" ]; then
+ cp /etc/trino/conf/hive-presto-master.keytab
/keytabs/other-hive-presto-master.keytab
+ cp /etc/trino/conf/presto-server.keytab /keytabs/other-presto-server.keytab
+else
+ echo "Invalid index parameter. Exiting."
+ exit 1
+fi
/usr/local/hadoop-run.sh &
# check healthy hear
echo "Waiting for hadoop to be healthy"
-for i in {1..10}; do
+for i in {1..60}; do
if /usr/local/health.sh; then
echo "Hadoop is healthy"
break
fi
- echo "Hadoop is not healthy yet. Retrying in 20 seconds..."
- sleep 20
+ echo "Hadoop is not healthy yet. Retrying in 60 seconds..."
+ sleep 5
done
-if [ $i -eq 10 ]; then
- echo "Hadoop did not become healthy after 120 attempts. Exiting."
+if [ $i -eq 60 ]; then
+ echo "Hadoop did not become healthy after 60 attempts. Exiting."
exit 1
fi
echo "Init kerberos test data"
-kinit -kt /etc/hive/conf/hive.keytab hive/[email protected]
+
+if [ "$1" == "1" ]; then
+ kinit -kt /etc/hive/conf/hive.keytab hive/[email protected]
+elif [ "$1" == "2" ]; then
+ kinit -kt /etc/hive/conf/hive.keytab hive/[email protected]
+else
+ echo "Invalid index parameter. Exiting."
+ exit 1
+fi
hive -f /usr/local/sql/create_kerberos_hive_table.sql
-touch /mnt/SUCCESS
-tail -f /dev/null
+exec_success_hook
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl
b/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl
new file mode 100644
index 00000000000..41c95057a92
--- /dev/null
+++ b/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl
@@ -0,0 +1,65 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:mysql://127.0.0.1:${MYSQL_PORT}/metastore
+HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=com.mysql.jdbc.Driver
+HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=root
+HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=root
+HIVE_SITE_CONF_datanucleus_autoCreateSchema=false
+HIVE_SITE_CONF_hive_metastore_port=${HMS_PORT}
+HIVE_SITE_CONF_hive_metastore_uris=thrift://${IP_HOST}:${HMS_PORT}
+HIVE_SITE_CONF_hive_server2_thrift_bind_host=0.0.0.0
+HIVE_SITE_CONF_hive_server2_thrift_port=${HS_PORT}
+HIVE_SITE_CONF_hive_server2_webui_port=0
+HIVE_SITE_CONF_hive_compactor_initiator_on=true
+HIVE_SITE_CONF_hive_compactor_worker_threads=2
+HIVE_SITE_CONF_metastore_storage_schema_reader_impl=org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader
+BEELINE_SITE_CONF_beeline_hs2_jdbc_url_tcpUrl=jdbc:hive2://${HOST}:${HS_PORT}/default;user=hdfs;password=hive
+BEELINE_SITE_CONF_beeline_hs2_jdbc_url_httpUrl=jdbc:hive2://${HOST}:${HS_PORT}/default;user=hdfs;password=hive
+
+
+CORE_CONF_fs_defaultFS=hdfs://${HOST}:${FS_PORT}
+CORE_CONF_hadoop_http_staticuser_user=root
+CORE_CONF_hadoop_proxyuser_hue_hosts=*
+CORE_CONF_hadoop_proxyuser_hue_groups=*
+
+HDFS_CONF_dfs_webhdfs_enabled=true
+HDFS_CONF_dfs_permissions_enabled=false
+HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false
+HDFS_CONF_dfs_datanode_address=${HOST}:${DFS_DN_PORT}
+HDFS_CONF_dfs_datanode_http_address=${HOST}:${DFS_DN_HTTP_PORT}
+HDFS_CONF_dfs_datanode_ipc_address=${HOST}:${DFS_DN_IPC_PORT}
+HDFS_CONF_dfs_namenode_http___address=${HOST}:${DFS_NN_HTTP_PORT}
+YARN_CONF_yarn_log___aggregation___enable=true
+YARN_CONF_yarn_resourcemanager_recovery_enabled=true
+YARN_CONF_yarn_resourcemanager_store_class=org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore
+YARN_CONF_yarn_resourcemanager_fs_state___store_uri=/rmstate
+YARN_CONF_yarn_nodemanager_remote___app___log___dir=/var/log/hadoop-yarn/apps
+YARN_CONF_yarn_log_server_url=http://${HOST}:${YARM_LOG_SERVER_PORT}/jobhistory/logs
+YARN_CONF_yarn_timeline___service_enabled=false
+YARN_CONF_yarn_timeline___service_generic___application___history_enabled=true
+YARN_CONF_yarn_resourcemanager_system___metrics___publisher_enabled=true
+YARN_CONF_yarn_resourcemanager_hostname=${HOST}
+MAPRED_CONF_mapreduce_shuffle_port=${MAPREDUCE_SHUFFLE_PORT}
+YARN_CONF_yarn_timeline___service_hostname=${HOST}
+YARN_CONF_yarn_resourcemanager_address=${HOST}:${YARN_RM_PORT}
+YARN_CONF_yarn_resourcemanager_scheduler_address=${HOST}:${YARN_RM_SCHEDULER_PORT}
+YARN_CONF_yarn_resourcemanager_resource___tracker_address=${HOST}:${YARN_RM_TRACKER_PORT}
+YARN_CONF_yarn_resourcemanager_admin_address=${HOST}:${YARN_RM_ADMIN_PORT}
+YARN_CONF_yarn_resourcemanager_webapp_address=${HOST}:${YARN_RM_WEBAPP_PORT}
+YARN_CONF_yarn_nodemanager_localizer_address=${HOST}:${YARN_NM_LOCAL_PORT}
+YARN_CONF_yarn_nodemanager_webapp_address=${HOST}:${YARN_NM_WEBAPP_PORT}
\ No newline at end of file
diff --git
a/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh
b/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh
index 473d7ceaeb6..515f37e36ac 100755
--- a/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh
+++ b/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh
@@ -32,4 +32,3 @@ if test -d "${HEALTH_D}"; then
"${health_script}" &>> /var/log/container-health.log || exit 1
done
fi
-exit 0
diff --git
a/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check-2.sh
b/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check-2.sh
index 854524dac1f..7545969bc47 100755
---
a/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check-2.sh
+++
b/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check-2.sh
@@ -17,4 +17,4 @@
# under the License.
kinit -kt /etc/hive/conf/hive.keytab hive/[email protected]
-beeline -u
"jdbc:hive2://localhost:10000/default;principal=hive/[email protected]"
-e "show databases;"
\ No newline at end of file
+beeline -u
"jdbc:hive2://localhost:16000/default;principal=hive/[email protected]"
-e "show databases;"
\ No newline at end of file
diff --git
a/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check.sh
b/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check.sh
index 4d3d86f69a2..ab464b5233b 100755
---
a/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check.sh
+++
b/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check.sh
@@ -17,4 +17,4 @@
# under the License.
kinit -kt /etc/hive/conf/hive.keytab hive/[email protected]
-beeline -u
"jdbc:hive2://localhost:10000/default;principal=hive/[email protected]"
-e "show databases;"
\ No newline at end of file
+beeline -u
"jdbc:hive2://localhost:15000/default;principal=hive/[email protected]"
-e "show databases;"
\ No newline at end of file
diff --git
a/docker/thirdparties/docker-compose/kerberos/health-checks/hadoop-health-check.sh
b/docker/thirdparties/docker-compose/kerberos/health-checks/supervisorctl-check.sh
similarity index 100%
rename from
docker/thirdparties/docker-compose/kerberos/health-checks/hadoop-health-check.sh
rename to
docker/thirdparties/docker-compose/kerberos/health-checks/supervisorctl-check.sh
diff --git a/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl
b/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl
index e635ed6bb27..9a1520b74db 100644
--- a/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl
+++ b/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl
@@ -16,68 +16,55 @@
# under the License.
version: "3"
services:
- hive-krb:
+ hive-krb1:
image: doristhirdpartydocker/trinodb:hdp3.1-hive-kerberized_96
- container_name: doris--kerberos1
+ container_name: doris-${CONTAINER_UID}-kerberos1
volumes:
+ - ../common:/usr/local/common
- ./two-kerberos-hives:/keytabs
- ./sql:/usr/local/sql
-
./common/hadoop/apply-config-overrides.sh:/etc/hadoop-init.d/00-apply-config-overrides.sh
- ./common/hadoop/hadoop-run.sh:/usr/local/hadoop-run.sh
- ./health-checks/health.sh:/usr/local/health.sh
- -
./health-checks/hadoop-health-check.sh:/etc/health.d/hadoop-health-check.sh
+ -
./health-checks/supervisorctl-check.sh:/etc/health.d/supervisorctl-check.sh
- ./health-checks/hive-health-check.sh:/etc/health.d/hive-health-check.sh
- ./entrypoint-hive-master.sh:/usr/local/entrypoint-hive-master.sh
+ - ./conf/kerberos1/my.cnf:/etc/my.cnf
+ - ./conf/kerberos1/kdc.conf:/var/kerberos/krb5kdc/kdc.conf
+ - ./conf/kerberos1/krb5.conf:/etc/krb5.conf
hostname: hadoop-master
- entrypoint: /usr/local/entrypoint-hive-master.sh
+ entrypoint: /usr/local/entrypoint-hive-master.sh 1
healthcheck:
- test: ["CMD", "ls", "/mnt/SUCCESS"]
- interval: 20s
- timeout: 60s
+ test: ["CMD", "ls", "/tmp/SUCCESS"]
+ interval: 5s
+ timeout: 10s
retries: 120
- ports:
- - "5806:5006"
- - "8820:8020"
- - "8842:8042"
- - "9800:9000"
- - "9883:9083"
- - "18000:10000"
- networks:
- doris--krb_net:
- ipv4_address: 172.31.71.25
-
+ network_mode: "host"
+ env_file:
+ - ./hadoop-hive-1.env
hive-krb2:
image: doristhirdpartydocker/trinodb:hdp3.1-hive-kerberized-2_96
- container_name: doris--kerberos2
+ container_name: doris-${CONTAINER_UID}-kerberos2
hostname: hadoop-master-2
volumes:
+ - ../common:/usr/local/common
- ./two-kerberos-hives:/keytabs
- ./sql:/usr/local/sql
-
./common/hadoop/apply-config-overrides.sh:/etc/hadoop-init.d/00-apply-config-overrides.sh
- ./common/hadoop/hadoop-run.sh:/usr/local/hadoop-run.sh
- ./health-checks/health.sh:/usr/local/health.sh
- -
./health-checks/hadoop-health-check.sh:/etc/health.d/hadoop-health-check.sh
+ -
./health-checks/supervisorctl-check.sh:/etc/health.d/supervisorctl-check.sh
-
./health-checks/hive-health-check-2.sh:/etc/health.d/hive-health-check-2.sh
- - ./entrypoint-hive-master-2.sh:/usr/local/entrypoint-hive-master-2.sh
- entrypoint: /usr/local/entrypoint-hive-master-2.sh
+ - ./entrypoint-hive-master.sh:/usr/local/entrypoint-hive-master.sh
+ - ./conf/kerberos2/my.cnf:/etc/my.cnf
+ - ./conf/kerberos2/kdc.conf:/var/kerberos/krb5kdc/kdc.conf
+ - ./conf/kerberos2/krb5.conf:/etc/krb5.conf
+ entrypoint: /usr/local/entrypoint-hive-master.sh 2
healthcheck:
- test: ["CMD", "ls", "/mnt/SUCCESS"]
- interval: 20s
- timeout: 60s
+ test: ["CMD", "ls", "/tmp/SUCCESS"]
+ interval: 5s
+ timeout: 10s
retries: 120
- ports:
- - "15806:5006"
- - "18820:8020"
- - "18842:8042"
- - "19800:9000"
- - "19883:9083"
- - "18800:10000"
- networks:
- doris--krb_net:
- ipv4_address: 172.31.71.26
-
-networks:
- doris--krb_net:
- ipam:
- config:
- - subnet: 172.31.71.0/24
+ network_mode: "host"
+ env_file:
+ - ./hadoop-hive-2.env
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env
b/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env
new file mode 100644
index 00000000000..aaf4c639fa8
--- /dev/null
+++ b/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env
@@ -0,0 +1,47 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Change this to a specific string.
+# Do not use "_" or other sepcial characters, only number and alphabeta.
+# NOTICE: change this uid will modify hive-*.yaml
+
+export HOST="hadoop-master"
+export FS_PORT=8520
+export HMS_PORT=9583
+export HS_PORT=15000
+export MYSQL_PORT=3356
+export DFS_DN_PORT=9566
+export DFS_DN_HTTP_PORT=9564
+export DFS_DN_IPC_PORT=9567
+export DFS_NN_HTTP_PORT=9570
+export YARM_LOG_SERVER_PORT=8588
+export YARN_RM_PORT=8532
+export YARN_RM_SCHEDULER_PORT=8530
+export YARN_RM_TRACKER_PORT=8531
+export YARN_RM_ADMIN_PORT=8533
+export YARN_RM_WEBAPP_PORT=8589
+export YARN_NM_LOCAL_PORT=8540
+export YARN_NM_WEBAPP_PORT=8542
+export MAPREDUCE_SHUFFLE_PORT=13562
+export KADMIND_PORT=5464
+export KDC_PORT1=5588
+export KDC_PORT2=5589
+export KADMIND_PORT1=5749
+export KADMIND_PORT2=5750
+export KPASSWD_PORT1=5464
+export KPASSWD_PORT2=5465
\ No newline at end of file
diff --git
a/docker/thirdparties/docker-compose/kerberos/ccache/create_kerberos_credential_cache_files.sh
b/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env
similarity index 50%
rename from
docker/thirdparties/docker-compose/kerberos/ccache/create_kerberos_credential_cache_files.sh
rename to docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env
index 2bba3f928b1..fca68d60162 100644
---
a/docker/thirdparties/docker-compose/kerberos/ccache/create_kerberos_credential_cache_files.sh
+++ b/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,18 +16,28 @@
# specific language governing permissions and limitations
# under the License.
-set -exuo pipefail
+# Change this to a specific string.
+# Do not use "_" or other sepcial characters, only number and alphabeta.
+# NOTICE: change this uid will modify hive-*.yaml
-TICKET_LIFETIME='30m'
-
-kinit -l "$TICKET_LIFETIME" -f -c /etc/trino/conf/presto-server-krbcc \
- -kt /etc/trino/conf/presto-server.keytab presto-server/$(hostname
-f)@LABS.TERADATA.COM
-
-kinit -l "$TICKET_LIFETIME" -f -c /etc/trino/conf/hive-presto-master-krbcc \
- -kt /etc/trino/conf/hive-presto-master.keytab hive/$(hostname
-f)@LABS.TERADATA.COM
-
-kinit -l "$TICKET_LIFETIME" -f -c /etc/trino/conf/hdfs-krbcc \
- -kt /etc/hadoop/conf/hdfs.keytab hdfs/[email protected]
-
-kinit -l "$TICKET_LIFETIME" -f -c /etc/trino/conf/hive-krbcc \
- -kt /etc/hive/conf/hive.keytab hive/[email protected]
+export HOST="hadoop-master-2"
+export FS_PORT=8620
+export HMS_PORT=9683
+export HS_PORT=16000
+export MYSQL_PORT=3366
+export DFS_DN_PORT=9666
+export DFS_DN_HTTP_PORT=9664
+export DFS_DN_IPC_PORT=9667
+export DFS_NN_HTTP_PORT=9670
+export YARM_LOG_SERVER_PORT=8688
+export YARN_RM_PORT=8632
+export YARN_RM_SCHEDULER_PORT=8630
+export YARN_RM_TRACKER_PORT=8631
+export YARN_RM_ADMIN_PORT=8633
+export YARN_RM_WEBAPP_PORT=8689
+export YARN_NM_LOCAL_PORT=8640
+export YARN_NM_WEBAPP_PORT=8642
+export MAPREDUCE_SHUFFLE_PORT=13662
+export KDC_PORT1=6688
+export KADMIND_PORT1=6749
+export KPASSWD_PORT1=6464
\ No newline at end of file
diff --git a/docker/thirdparties/run-thirdparties-docker.sh
b/docker/thirdparties/run-thirdparties-docker.sh
index e3769025fec..a581baa6cfa 100755
--- a/docker/thirdparties/run-thirdparties-docker.sh
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@@ -582,8 +582,22 @@ start_lakesoul() {
start_kerberos() {
echo "RUN_KERBEROS"
- cp "${ROOT}"/docker-compose/kerberos/kerberos.yaml.tpl
"${ROOT}"/docker-compose/kerberos/kerberos.yaml
- sed -i "s/doris--/${CONTAINER_UID}/g"
"${ROOT}"/docker-compose/kerberos/kerberos.yaml
+ eth_name=$(ifconfig -a | grep -E "^eth[0-9]" | sort -k1.4n | awk -F ':'
'{print $1}' | head -n 1)
+ IP_HOST=$(ifconfig "${eth_name}" | grep inet | grep -v 127.0.0.1 | grep -v
inet6 | awk '{print $2}' | tr -d "addr:" | head -n 1)
+ export IP_HOST=${IP_HOST}
+ export CONTAINER_UID=${CONTAINER_UID}
+ envsubst <"${ROOT}"/docker-compose/kerberos/kerberos.yaml.tpl
>"${ROOT}"/docker-compose/kerberos/kerberos.yaml
+ for i in {1..2}; do
+ . "${ROOT}"/docker-compose/kerberos/kerberos${i}_settings.env
+ envsubst <"${ROOT}"/docker-compose/kerberos/hadoop-hive.env.tpl
>"${ROOT}"/docker-compose/kerberos/hadoop-hive-${i}.env
+ envsubst <"${ROOT}"/docker-compose/kerberos/conf/my.cnf.tpl >
"${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/my.cnf
+ envsubst
<"${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/kdc.conf.tpl >
"${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/kdc.conf
+ envsubst
<"${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/krb5.conf.tpl >
"${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/krb5.conf
+ done
+ sudo chmod a+w /etc/hosts
+ sudo sed -i "1i${IP_HOST} hadoop-master" /etc/hosts
+ sudo sed -i "1i${IP_HOST} hadoop-master-2" /etc/hosts
+ sudo cp "${ROOT}"/docker-compose/kerberos/kerberos.yaml.tpl
"${ROOT}"/docker-compose/kerberos/kerberos.yaml
sudo docker compose -f "${ROOT}"/docker-compose/kerberos/kerberos.yaml down
sudo rm -rf "${ROOT}"/docker-compose/kerberos/data
if [[ "${STOP}" -ne 1 ]]; then
@@ -591,15 +605,11 @@ start_kerberos() {
rm -rf "${ROOT}"/docker-compose/kerberos/two-kerberos-hives/*.keytab
rm -rf "${ROOT}"/docker-compose/kerberos/two-kerberos-hives/*.jks
rm -rf "${ROOT}"/docker-compose/kerberos/two-kerberos-hives/*.conf
- sudo docker compose -f "${ROOT}"/docker-compose/kerberos/kerberos.yaml
up -d
+ sudo docker compose -f "${ROOT}"/docker-compose/kerberos/kerberos.yaml
up -d --wait
sudo rm -f /keytabs
sudo ln -s "${ROOT}"/docker-compose/kerberos/two-kerberos-hives
/keytabs
sudo cp "${ROOT}"/docker-compose/kerberos/common/conf/doris-krb5.conf
/keytabs/krb5.conf
sudo cp "${ROOT}"/docker-compose/kerberos/common/conf/doris-krb5.conf
/etc/krb5.conf
-
- sudo chmod a+w /etc/hosts
- echo '172.31.71.25 hadoop-master' >> /etc/hosts
- echo '172.31.71.26 hadoop-master-2' >> /etc/hosts
sleep 2
fi
}
diff --git
a/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy
b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy
index 505c5208c99..e1612bcf884 100644
---
a/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy
+++
b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy
@@ -25,6 +25,7 @@ suite("test_single_hive_kerberos",
"p0,external,kerberos,external_docker,externa
println "Docker containers:"
println output
String enabled = context.config.otherConfigs.get("enableKerberosTest")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
if (enabled != null && enabled.equalsIgnoreCase("true")) {
String hms_catalog_name = "test_single_hive_kerberos"
sql """drop catalog if exists hms_kerberos;"""
@@ -32,8 +33,8 @@ suite("test_single_hive_kerberos",
"p0,external,kerberos,external_docker,externa
CREATE CATALOG IF NOT EXISTS hms_kerberos
PROPERTIES (
"type" = "hms",
- "hive.metastore.uris" = "thrift://172.31.71.25:9083",
- "fs.defaultFS" = "hdfs://172.31.71.25:8020",
+ "hive.metastore.uris" = "thrift://${externalEnvIp}:9583",
+ "fs.defaultFS" = "hdfs://${externalEnvIp}:8520",
"hadoop.security.authentication" = "kerberos",
"hadoop.kerberos.principal"="presto-server/[email protected]",
"hadoop.kerberos.keytab" = "/keytabs/presto-server.keytab",
@@ -42,7 +43,7 @@ suite("test_single_hive_kerberos",
"p0,external,kerberos,external_docker,externa
RULE:[2:\$1@\$0](.*@OTHERREALM.COM)s/@.*//
DEFAULT",
"hive.metastore.sasl.enabled " = "true",
- "hive.metastore.kerberos.principal" =
"hive/[email protected]"
+ "hive.metastore.kerberos.principal" =
"hive/[email protected]"
);
"""
sql """ switch hms_kerberos """
@@ -56,8 +57,8 @@ suite("test_single_hive_kerberos",
"p0,external,kerberos,external_docker,externa
CREATE CATALOG IF NOT EXISTS hms_kerberos_hadoop_err1
PROPERTIES (
"type" = "hms",
- "hive.metastore.uris" = "thrift://172.31.71.25:9083",
- "fs.defaultFS" = "hdfs://172.31.71.25:8020",
+ "hive.metastore.uris" = "thrift://${externalEnvIp}:9583",
+ "fs.defaultFS" = "hdfs://${externalEnvIp}:8520",
"hadoop.security.authentication" = "kerberos",
"hadoop.kerberos.principal"="presto-server/[email protected]",
"hadoop.kerberos.keytab" = "/keytabs/presto-server.keytab"
@@ -79,8 +80,8 @@ suite("test_single_hive_kerberos",
"p0,external,kerberos,external_docker,externa
PROPERTIES (
"type" = "hms",
"hive.metastore.sasl.enabled " = "true",
- "hive.metastore.uris" = "thrift://172.31.71.25:9083",
- "fs.defaultFS" = "hdfs://172.31.71.25:8020"
+ "hive.metastore.uris" = "thrift://${externalEnvIp}:9583",
+ "fs.defaultFS" = "hdfs://${externalEnvIp}:8520"
);
"""
sql """ switch hms_kerberos_hadoop_err2 """
@@ -95,8 +96,8 @@ suite("test_single_hive_kerberos",
"p0,external,kerberos,external_docker,externa
// CREATE CATALOG IF NOT EXISTS hms_keberos_ccache
// PROPERTIES (
// "type" = "hms",
- // "hive.metastore.uris" =
"thrift://172.31.71.25:9083",
- // "fs.defaultFS" = "hdfs://172.31.71.25:8020",
+ // "hive.metastore.uris" =
"thrift://${externalEnvIp}:9583",
+ // "fs.defaultFS" = "hdfs://${externalEnvIp}:8520",
// "hadoop.security.authentication" = "kerberos",
//
"hadoop.kerberos.principal"="presto-server/[email protected]",
// "hadoop.kerberos.keytab" =
"/keytabs/presto-server.keytab",
diff --git
a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy
b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy
index 29b0cb2cd7e..764078c62a6 100644
---
a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy
+++
b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy
@@ -29,6 +29,7 @@ suite("test_two_hive_kerberos",
"p0,external,kerberos,external_docker,external_d
println "Docker containers:"
println output
String enabled = context.config.otherConfigs.get("enableKerberosTest")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
if (enabled != null && enabled.equalsIgnoreCase("true")) {
String hms_catalog_name = "test_two_hive_kerberos"
sql """drop catalog if exists ${hms_catalog_name};"""
@@ -36,14 +37,14 @@ suite("test_two_hive_kerberos",
"p0,external,kerberos,external_docker,external_d
CREATE CATALOG IF NOT EXISTS ${hms_catalog_name}
PROPERTIES (
"type" = "hms",
- "hive.metastore.uris" = "thrift://172.31.71.25:9083",
- "fs.defaultFS" = "hdfs://172.31.71.25:8020",
+ "hive.metastore.uris" = "thrift://${externalEnvIp}:9583",
+ "fs.defaultFS" = "hdfs://${externalEnvIp}:8520",
"hadoop.kerberos.min.seconds.before.relogin" = "5",
"hadoop.security.authentication" = "kerberos",
"hadoop.kerberos.principal"="hive/[email protected]",
"hadoop.kerberos.keytab" =
"/keytabs/hive-presto-master.keytab",
"hive.metastore.sasl.enabled " = "true",
- "hive.metastore.kerberos.principal" =
"hive/[email protected]"
+ "hive.metastore.kerberos.principal" =
"hive/[email protected]"
);
"""
@@ -52,14 +53,14 @@ suite("test_two_hive_kerberos",
"p0,external,kerberos,external_docker,external_d
CREATE CATALOG IF NOT EXISTS other_${hms_catalog_name}
PROPERTIES (
"type" = "hms",
- "hive.metastore.uris" = "thrift://172.31.71.26:9083",
- "fs.defaultFS" = "hdfs://172.31.71.26:8020",
+ "hive.metastore.uris" = "thrift://${externalEnvIp}:9683",
+ "fs.defaultFS" = "hdfs://${externalEnvIp}:8620",
"hadoop.kerberos.min.seconds.before.relogin" = "5",
"hadoop.security.authentication" = "kerberos",
"hadoop.kerberos.principal"="hive/[email protected]",
"hadoop.kerberos.keytab" =
"/keytabs/other-hive-presto-master.keytab",
"hive.metastore.sasl.enabled " = "true",
- "hive.metastore.kerberos.principal" =
"hive/[email protected]",
+ "hive.metastore.kerberos.principal" =
"hive/[email protected]",
"hadoop.security.auth_to_local"
="RULE:[2:\$1@\$0](.*@OTHERREALM.COM)s/@.*//
RULE:[2:\$1@\$0](.*@OTHERLABS.TERADATA.COM)s/@.*//
DEFAULT"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
