This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new beb0192228c branch-3.0: [feat](doris compose) diff user use diff start
subnet #41996 (#43454)
beb0192228c is described below
commit beb0192228c5938c471e3decb9a93a7620dbd0af
Author: yujun <[email protected]>
AuthorDate: Fri Nov 8 09:46:12 2024 +0800
branch-3.0: [feat](doris compose) diff user use diff start subnet #41996
(#43454)
If users use different LOCAL_DORIS_PATH, their clusters' network maybe
conflict. So let different user use different searched start subnet.
cherry-pick: #41996
---
docker/runtime/doris-compose/cluster.py | 45 +++++++++++++++++++---
docker/runtime/doris-compose/command.py | 3 +-
docker/runtime/doris-compose/resource/init_be.sh | 2 +-
.../runtime/doris-compose/resource/init_cloud.sh | 5 +--
docker/runtime/doris-compose/resource/init_fe.sh | 15 +++++---
5 files changed, 53 insertions(+), 17 deletions(-)
diff --git a/docker/runtime/doris-compose/cluster.py
b/docker/runtime/doris-compose/cluster.py
index 9875f8aa4e9..ba834167bd1 100644
--- a/docker/runtime/doris-compose/cluster.py
+++ b/docker/runtime/doris-compose/cluster.py
@@ -17,6 +17,8 @@
import configparser
import filelock
+import getpass
+import hashlib
import jsonpickle
import os
import os.path
@@ -24,7 +26,10 @@ import utils
DOCKER_DORIS_PATH = "/opt/apache-doris"
LOCAL_DORIS_PATH = os.getenv("LOCAL_DORIS_PATH", "/tmp/doris")
-DORIS_SUBNET_START = int(os.getenv("DORIS_SUBNET_START", 128))
+
+# an integer between 128 and 191, generally no need to set
+DORIS_SUBNET_START = os.getenv("DORIS_SUBNET_START")
+
LOCAL_RESOURCE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"resource")
DOCKER_RESOURCE_PATH = os.path.join(DOCKER_DORIS_PATH, "resource")
@@ -95,11 +100,39 @@ def gen_subnet_prefix16():
except:
pass
- for i in range(DORIS_SUBNET_START, 192):
- for j in range(256):
- subnet = "{}.{}".format(i, j)
- if not used_subnet.get(subnet, False):
- return subnet
+ subnet_begin = 128
+ subnet_end = 192
+
+ subnet_part_1 = None
+ subnet_part_2 = None
+ if DORIS_SUBNET_START:
+ subnet_part_1 = int(DORIS_SUBNET_START)
+ subnet_part_2 = 0
+ else:
+ m = hashlib.md5()
+ m.update(getpass.getuser().encode("utf-8"))
+ hash_val = int(m.hexdigest(), 16)
+ # want subnet part ii to be a small num, just less than 100, so don't
use 256 here.
+ small_width = 100
+ slot_num = (subnet_end - subnet_begin) * small_width
+ idx = hash_val % slot_num
+ if idx < 0:
+ idx += slot_num
+ subnet_part_1 = subnet_begin + int(idx / small_width)
+ subnet_part_2 = idx % small_width
+
+ intervals = [
+ [(subnet_part_1, subnet_part_1 + 1), (subnet_part_2, 256)],
+ [(subnet_part_1 + 1, subnet_end), (0, 256)],
+ [(subnet_begin, subnet_part_1), (0, 256)],
+ [(subnet_part_1, subnet_part_1 + 1), (0, subnet_part_2)],
+ ]
+ for interval in intervals:
+ for i in range(interval[0][0], interval[0][1]):
+ for j in range(interval[1][0], interval[1][1]):
+ subnet = "{}.{}".format(i, j)
+ if not used_subnet.get(subnet, False):
+ return subnet
raise Exception("Failed to gen subnet")
diff --git a/docker/runtime/doris-compose/command.py
b/docker/runtime/doris-compose/command.py
index 1217c34ac45..7a2f3f3c195 100644
--- a/docker/runtime/doris-compose/command.py
+++ b/docker/runtime/doris-compose/command.py
@@ -223,7 +223,6 @@ class SimpleCommand(Command):
LOG.info(
utils.render_green("{} succ, total related node num {}".format(
show_cmd, related_node_num)))
- return ""
if for_all:
related_nodes = cluster.get_all_nodes()
@@ -381,7 +380,7 @@ class UpCommand(Command):
group2.add_argument("--force-recreate",
default=False,
action=self._get_parser_bool_action(True),
- help="Recreate containers even if their
configuration" \
+ help="Recreate containers even if their
configuration " \
"and image haven't changed. ")
parser.add_argument("--coverage-dir",
diff --git a/docker/runtime/doris-compose/resource/init_be.sh
b/docker/runtime/doris-compose/resource/init_be.sh
index e0ed5b92cb8..08cc914f6af 100755
--- a/docker/runtime/doris-compose/resource/init_be.sh
+++ b/docker/runtime/doris-compose/resource/init_be.sh
@@ -173,7 +173,7 @@ main() {
add_be_to_cluster
health_log "run start_be.sh"
- bash $DORIS_HOME/bin/start_be.sh --daemon
+ bash $DORIS_HOME/bin/start_be.sh --daemon | tee -a $DORIS_HOME/log/be.out
wait_process
}
diff --git a/docker/runtime/doris-compose/resource/init_cloud.sh
b/docker/runtime/doris-compose/resource/init_cloud.sh
index 5740335ace3..18dfc4430e2 100644
--- a/docker/runtime/doris-compose/resource/init_cloud.sh
+++ b/docker/runtime/doris-compose/resource/init_cloud.sh
@@ -116,9 +116,8 @@ main() {
check_init_cloud &
- health_log "input args: $ARGS"
-
- bash bin/start.sh $ARGS --daemon
+ health_log "run starts.sh with args: $ARGS"
+ bash bin/start.sh $ARGS --daemon | tee -a $DORIS_HOME/log/doris_cloud.out
wait_process
}
diff --git a/docker/runtime/doris-compose/resource/init_fe.sh
b/docker/runtime/doris-compose/resource/init_fe.sh
index 39d3ed3fa93..b69ac3a209e 100755
--- a/docker/runtime/doris-compose/resource/init_fe.sh
+++ b/docker/runtime/doris-compose/resource/init_fe.sh
@@ -81,10 +81,15 @@ fe_daemon() {
done
}
+run_fe() {
+ health_log "run start_fe.sh"
+ bash $DORIS_HOME/bin/start_fe.sh --daemon $@ | tee -a
$DORIS_HOME/log/fe.out
+}
+
start_cloud_fe() {
if [ -f "$REGISTER_FILE" ]; then
fe_daemon &
- bash $DORIS_HOME/bin/start_fe.sh --daemon
+ run_fe
return
fi
@@ -95,7 +100,7 @@ start_cloud_fe() {
touch $REGISTER_FILE
fe_daemon &
- bash $DORIS_HOME/bin/start_fe.sh --daemon
+ run_fe
if [ "$MY_ID" == "1" ]; then
echo $MY_IP >$MASTER_FE_IP_FILE
@@ -162,7 +167,7 @@ start_cloud_fe() {
touch $REGISTER_FILE
fe_daemon &
- bash $DORIS_HOME/bin/start_fe.sh --daemon
+ run_fe
if [ "$MY_ID" == "1" ]; then
echo $MY_IP >$MASTER_FE_IP_FILE
@@ -199,11 +204,11 @@ start_local_fe() {
if [ -f $REGISTER_FILE ]; then
fe_daemon &
- bash $DORIS_HOME/bin/start_fe.sh --daemon
+ run_fe
else
add_local_fe
fe_daemon &
- bash $DORIS_HOME/bin/start_fe.sh --helper
$MASTER_FE_IP:$FE_EDITLOG_PORT --daemon
+ run_fe --helper $MASTER_FE_IP:$FE_EDITLOG_PORT
fi
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
