This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new b0922790c0 HDDS-8767. Convert s3-haproxy to an add-on (#4835)
b0922790c0 is described below
commit b0922790c0cded303507d0d4d572c8e5f6538236
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Wed Jun 7 22:28:55 2023 +0200
HDDS-8767. Convert s3-haproxy to an add-on (#4835)
---
.../haproxy.cfg => common/s3-haproxy.cfg} | 0
.../.env => common/s3-haproxy.yaml} | 35 ++++++++-
.../test.sh => ozone/test-s3-haproxy.sh} | 1 +
.../compose/ozones3-haproxy/docker-compose.yaml | 86 ----------------------
.../src/main/compose/ozones3-haproxy/docker-config | 42 -----------
5 files changed, 33 insertions(+), 131 deletions(-)
diff --git
a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/haproxy-conf/haproxy.cfg
b/hadoop-ozone/dist/src/main/compose/common/s3-haproxy.cfg
similarity index 100%
rename from
hadoop-ozone/dist/src/main/compose/ozones3-haproxy/haproxy-conf/haproxy.cfg
rename to hadoop-ozone/dist/src/main/compose/common/s3-haproxy.cfg
diff --git a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/.env
b/hadoop-ozone/dist/src/main/compose/common/s3-haproxy.yaml
similarity index 56%
rename from hadoop-ozone/dist/src/main/compose/ozones3-haproxy/.env
rename to hadoop-ozone/dist/src/main/compose/common/s3-haproxy.yaml
index 140975d4bd..abc9011fb6 100644
--- a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/.env
+++ b/hadoop-ozone/dist/src/main/compose/common/s3-haproxy.yaml
@@ -14,6 +14,35 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-HDDS_VERSION=${hdds.version}
-OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
-OZONE_RUNNER_IMAGE=apache/ozone-runner
+version: "3.4"
+
+x-s3-worker:
+ &s3-worker
+ image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
+ volumes:
+ - ../..:/opt/hadoop
+ env_file:
+ - docker-config
+ command: ["ozone","s3g"]
+
+services:
+ s3g:
+ image: haproxy:lts-alpine
+ volumes:
+ - ../..:/opt/hadoop
+ - ../common/s3-haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg
+ ports:
+ - 9878:9878
+ command: ["haproxy", "-f", "/usr/local/etc/haproxy/haproxy.cfg"]
+ s3g1:
+ <<: *s3-worker
+ ports:
+ - 9879:9878
+ s3g2:
+ <<: *s3-worker
+ ports:
+ - 9880:9878
+ s3g3:
+ <<: *s3-worker
+ ports:
+ - 9881:9878
diff --git a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/test.sh
b/hadoop-ozone/dist/src/main/compose/ozone/test-s3-haproxy.sh
similarity index 94%
rename from hadoop-ozone/dist/src/main/compose/ozones3-haproxy/test.sh
rename to hadoop-ozone/dist/src/main/compose/ozone/test-s3-haproxy.sh
index 238e7577a7..6a4ebff3f9 100755
--- a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone/test-s3-haproxy.sh
@@ -19,6 +19,7 @@ COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"
>/dev/null 2>&1 && pwd )"
export COMPOSE_DIR
export SECURITY_ENABLED=false
+export COMPOSE_FILE=docker-compose.yaml:../common/s3-haproxy.yaml
# shellcheck source=/dev/null
source "$COMPOSE_DIR/../testlib.sh"
diff --git
a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml
b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml
deleted file mode 100644
index 1809457c8d..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml
+++ /dev/null
@@ -1,86 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
- s3g:
- image: haproxy:latest
- volumes:
- - ../..:/opt/hadoop
- - ./haproxy-conf/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg
- ports:
- - 9878:9878
- datanode:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9864
- command: ["ozone","datanode"]
- env_file:
- - ./docker-config
- om:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9874:9874
- - 9862:9862
- environment:
- ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
- env_file:
- - ./docker-config
- command: ["ozone","om"]
- scm:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9876:9876
- - 9860:9860
- env_file:
- - ./docker-config
- environment:
- ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- OZONE-SITE.XML_hdds.scm.safemode.min.datanode:
"${OZONE_SAFEMODE_MIN_DATANODES:-1}"
- command: ["ozone","scm"]
- s3g1:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9879:9878
- env_file:
- - ./docker-config
- command: ["ozone","s3g"]
- s3g2:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9880:9878
- env_file:
- - ./docker-config
- command: ["ozone","s3g"]
- s3g3:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9881:9878
- env_file:
- - ./docker-config
- command: ["ozone","s3g"]
diff --git a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config
b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config
deleted file mode 100644
index 290a163ef8..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config
+++ /dev/null
@@ -1,42 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-CORE-SITE.XML_fs.defaultFS=ofs://om
-
-OZONE-SITE.XML_ozone.om.address=om
-OZONE-SITE.XML_ozone.scm.container.size=1GB
-OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
-OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
-OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_ozone.replication=3
-OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
-OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
-
-HDFS-SITE.XML_rpc.metrics.quantile.enable=true
-HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-
-OZONE_CONF_DIR=/etc/hadoop
-OZONE_LOG_DIR=/var/log/hadoop
-
-#Enable this variable to print out all hadoop rpc traffic to the stdout. See
http://byteman.jboss.org/ to define your own instrumentation.
-#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
-
-no_proxy=om,scm,s3g,recon,kdc,localhost,127.0.0.1
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
