This is an automated email from the ASF dual-hosted git repository. dataroaring pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push: new cd1b10e2273 branch-3.0: [fix](hive docker)Reserve host port for hive2 namenode and datanode #47262 (#47304) cd1b10e2273 is described below commit cd1b10e2273a248cf8ff766645e891e192dde9e9 Author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> AuthorDate: Wed Jan 22 15:13:55 2025 +0800 branch-3.0: [fix](hive docker)Reserve host port for hive2 namenode and datanode #47262 (#47304) Cherry-picked from #47262 Co-authored-by: Thearas <gaozif...@selectdb.com> --- .../docker-compose/hive/hive-2x.yaml.tpl | 9 +++++--- .../docker-compose/hive/hive-2x_settings.env | 2 +- .../docker-compose/hive/hive-3x.yaml.tpl | 9 +++++--- docker/thirdparties/run-thirdparties-docker.sh | 24 +++++++++++++++++++++- 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl index acc253e7027..29a44973284 100644 --- a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl +++ b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl @@ -63,8 +63,10 @@ services: expose: - "${HS_PORT}" depends_on: - - datanode - - namenode + datanode: + condition: service_healthy + namenode: + condition: service_healthy healthcheck: test: beeline -u "jdbc:hive2://127.0.0.1:${HS_PORT}/default" -n health_check -e "show databases;" interval: 10s @@ -86,7 +88,8 @@ services: volumes: - ./scripts:/mnt/scripts depends_on: - - hive-metastore-postgresql + hive-metastore-postgresql: + condition: service_healthy healthcheck: test: ["CMD", "sh", "-c", "/mnt/scripts/healthy_check.sh"] interval: 20s diff --git a/docker/thirdparties/docker-compose/hive/hive-2x_settings.env b/docker/thirdparties/docker-compose/hive/hive-2x_settings.env index ddd4625aaab..9045bb91683 100644 --- a/docker/thirdparties/docker-compose/hive/hive-2x_settings.env +++ b/docker/thirdparties/docker-compose/hive/hive-2x_settings.env @@ -20,7 +20,7 @@ # Do not use "_" or other sepcial characters, only number and alphabeta. # NOTICE: change this uid will modify hive-*.yaml -export FS_PORT=8020 # should be same as hive3HmsPort in regression-conf.groovy +export FS_PORT=8020 # should be same as hive2HmsPort in regression-conf.groovy export HMS_PORT=9083 # should be same as hive2HmsPort in regression-conf.groovy export HS_PORT=10000 # should be same as hive2ServerPort in regression-conf.groovy export PG_PORT=5432 # should be same as hive2PgPort in regression-conf.groovy diff --git a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl index 1f6eca76d43..c5b54bae032 100644 --- a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl +++ b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl @@ -63,8 +63,10 @@ services: expose: - "${HS_PORT}" depends_on: - - datanode - - namenode + datanode: + condition: service_healthy + namenode: + condition: service_healthy healthcheck: test: beeline -u "jdbc:hive2://127.0.0.1:${HS_PORT}/default" -n health_check -e "show databases;" interval: 10s @@ -86,7 +88,8 @@ services: volumes: - ./scripts:/mnt/scripts depends_on: - - hive-metastore-postgresql + hive-metastore-postgresql: + condition: service_healthy healthcheck: test: ["CMD", "sh", "-c", "/mnt/scripts/healthy_check.sh"] interval: 20s diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh index 34aba3aeb3b..755f780e7f4 100755 --- a/docker/thirdparties/run-thirdparties-docker.sh +++ b/docker/thirdparties/run-thirdparties-docker.sh @@ -33,8 +33,9 @@ Usage: $0 <options> [no option] start all components --help,-h show this usage -c mysql start MySQL - -c mysql,hive3 start MySQL and Hive3 + -c mysql,hive3 start MySQL and Hive3 --stop stop the specified components + --reserve-ports reserve host ports by setting 'net.ipv4.ip_local_reserved_ports' to avoid port already bind error All valid components: mysql,pg,oracle,sqlserver,clickhouse,es,hive2,hive3,iceberg,hudi,trino,kafka,mariadb,db2,oceanbase,lakesoul,kerberos @@ -44,6 +45,7 @@ Usage: $0 <options> COMPONENTS=$2 HELP=0 STOP=0 +NEED_RESERVE_PORTS=0 if ! OPTS="$(getopt \ -n "$0" \ @@ -79,6 +81,10 @@ else COMPONENTS=$2 shift 2 ;; + --reserve-ports) + NEED_RESERVE_PORTS=1 + shift + ;; --) shift break @@ -139,6 +145,8 @@ RUN_OCENABASE=0 RUN_LAKESOUL=0 RUN_KERBEROS=0 +RESERVED_PORTS="" + for element in "${COMPONENTS_ARR[@]}"; do if [[ "${element}"x == "mysql"x ]]; then RUN_MYSQL=1 @@ -154,6 +162,7 @@ for element in "${COMPONENTS_ARR[@]}"; do RUN_ES=1 elif [[ "${element}"x == "hive2"x ]]; then RUN_HIVE2=1 + RESERVED_PORTS="${RESERVED_PORTS},50070,50075" # namenode and datanode ports elif [[ "${element}"x == "hive3"x ]]; then RUN_HIVE3=1 elif [[ "${element}"x == "kafka"x ]]; then @@ -182,6 +191,17 @@ for element in "${COMPONENTS_ARR[@]}"; do fi done +reserve_ports() { + if [[ "${NEED_RESERVE_PORTS}" -eq 0 ]]; then + return + fi + + if [[ "${RESERVED_PORTS}"x != ""x ]]; then + echo "Reserve ports: ${RESERVED_PORTS}" + sudo sysctl -w net.ipv4.ip_local_reserved_ports="${RESERVED_PORTS}" + fi +} + start_es() { # elasticsearch cp "${ROOT}"/docker-compose/elasticsearch/es.yaml.tpl "${ROOT}"/docker-compose/elasticsearch/es.yaml @@ -613,6 +633,8 @@ start_kerberos() { echo "starting dockers in parallel" +reserve_ports + declare -A pids if [[ "${RUN_ES}" -eq 1 ]]; then --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org