This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new cd1b10e2273 branch-3.0: [fix](hive docker)Reserve host port for hive2
namenode and datanode #47262 (#47304)
cd1b10e2273 is described below
commit cd1b10e2273a248cf8ff766645e891e192dde9e9
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Wed Jan 22 15:13:55 2025 +0800
branch-3.0: [fix](hive docker)Reserve host port for hive2 namenode and
datanode #47262 (#47304)
Cherry-picked from #47262
Co-authored-by: Thearas <[email protected]>
---
.../docker-compose/hive/hive-2x.yaml.tpl | 9 +++++---
.../docker-compose/hive/hive-2x_settings.env | 2 +-
.../docker-compose/hive/hive-3x.yaml.tpl | 9 +++++---
docker/thirdparties/run-thirdparties-docker.sh | 24 +++++++++++++++++++++-
4 files changed, 36 insertions(+), 8 deletions(-)
diff --git a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
index acc253e7027..29a44973284 100644
--- a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
@@ -63,8 +63,10 @@ services:
expose:
- "${HS_PORT}"
depends_on:
- - datanode
- - namenode
+ datanode:
+ condition: service_healthy
+ namenode:
+ condition: service_healthy
healthcheck:
test: beeline -u "jdbc:hive2://127.0.0.1:${HS_PORT}/default" -n
health_check -e "show databases;"
interval: 10s
@@ -86,7 +88,8 @@ services:
volumes:
- ./scripts:/mnt/scripts
depends_on:
- - hive-metastore-postgresql
+ hive-metastore-postgresql:
+ condition: service_healthy
healthcheck:
test: ["CMD", "sh", "-c", "/mnt/scripts/healthy_check.sh"]
interval: 20s
diff --git a/docker/thirdparties/docker-compose/hive/hive-2x_settings.env
b/docker/thirdparties/docker-compose/hive/hive-2x_settings.env
index ddd4625aaab..9045bb91683 100644
--- a/docker/thirdparties/docker-compose/hive/hive-2x_settings.env
+++ b/docker/thirdparties/docker-compose/hive/hive-2x_settings.env
@@ -20,7 +20,7 @@
# Do not use "_" or other sepcial characters, only number and alphabeta.
# NOTICE: change this uid will modify hive-*.yaml
-export FS_PORT=8020 # should be same as hive3HmsPort in regression-conf.groovy
+export FS_PORT=8020 # should be same as hive2HmsPort in regression-conf.groovy
export HMS_PORT=9083 # should be same as hive2HmsPort in regression-conf.groovy
export HS_PORT=10000 # should be same as hive2ServerPort in
regression-conf.groovy
export PG_PORT=5432 # should be same as hive2PgPort in regression-conf.groovy
diff --git a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
index 1f6eca76d43..c5b54bae032 100644
--- a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
@@ -63,8 +63,10 @@ services:
expose:
- "${HS_PORT}"
depends_on:
- - datanode
- - namenode
+ datanode:
+ condition: service_healthy
+ namenode:
+ condition: service_healthy
healthcheck:
test: beeline -u "jdbc:hive2://127.0.0.1:${HS_PORT}/default" -n
health_check -e "show databases;"
interval: 10s
@@ -86,7 +88,8 @@ services:
volumes:
- ./scripts:/mnt/scripts
depends_on:
- - hive-metastore-postgresql
+ hive-metastore-postgresql:
+ condition: service_healthy
healthcheck:
test: ["CMD", "sh", "-c", "/mnt/scripts/healthy_check.sh"]
interval: 20s
diff --git a/docker/thirdparties/run-thirdparties-docker.sh
b/docker/thirdparties/run-thirdparties-docker.sh
index 34aba3aeb3b..755f780e7f4 100755
--- a/docker/thirdparties/run-thirdparties-docker.sh
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@@ -33,8 +33,9 @@ Usage: $0 <options>
[no option] start all components
--help,-h show this usage
-c mysql start MySQL
- -c mysql,hive3 start MySQL and Hive3
+ -c mysql,hive3 start MySQL and Hive3
--stop stop the specified components
+ --reserve-ports reserve host ports by setting
'net.ipv4.ip_local_reserved_ports' to avoid port already bind error
All valid components:
mysql,pg,oracle,sqlserver,clickhouse,es,hive2,hive3,iceberg,hudi,trino,kafka,mariadb,db2,oceanbase,lakesoul,kerberos
@@ -44,6 +45,7 @@ Usage: $0 <options>
COMPONENTS=$2
HELP=0
STOP=0
+NEED_RESERVE_PORTS=0
if ! OPTS="$(getopt \
-n "$0" \
@@ -79,6 +81,10 @@ else
COMPONENTS=$2
shift 2
;;
+ --reserve-ports)
+ NEED_RESERVE_PORTS=1
+ shift
+ ;;
--)
shift
break
@@ -139,6 +145,8 @@ RUN_OCENABASE=0
RUN_LAKESOUL=0
RUN_KERBEROS=0
+RESERVED_PORTS=""
+
for element in "${COMPONENTS_ARR[@]}"; do
if [[ "${element}"x == "mysql"x ]]; then
RUN_MYSQL=1
@@ -154,6 +162,7 @@ for element in "${COMPONENTS_ARR[@]}"; do
RUN_ES=1
elif [[ "${element}"x == "hive2"x ]]; then
RUN_HIVE2=1
+ RESERVED_PORTS="${RESERVED_PORTS},50070,50075" # namenode and datanode
ports
elif [[ "${element}"x == "hive3"x ]]; then
RUN_HIVE3=1
elif [[ "${element}"x == "kafka"x ]]; then
@@ -182,6 +191,17 @@ for element in "${COMPONENTS_ARR[@]}"; do
fi
done
+reserve_ports() {
+ if [[ "${NEED_RESERVE_PORTS}" -eq 0 ]]; then
+ return
+ fi
+
+ if [[ "${RESERVED_PORTS}"x != ""x ]]; then
+ echo "Reserve ports: ${RESERVED_PORTS}"
+ sudo sysctl -w net.ipv4.ip_local_reserved_ports="${RESERVED_PORTS}"
+ fi
+}
+
start_es() {
# elasticsearch
cp "${ROOT}"/docker-compose/elasticsearch/es.yaml.tpl
"${ROOT}"/docker-compose/elasticsearch/es.yaml
@@ -613,6 +633,8 @@ start_kerberos() {
echo "starting dockers in parallel"
+reserve_ports
+
declare -A pids
if [[ "${RUN_ES}" -eq 1 ]]; then
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
