This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 89048051c0d [enhance](S3) Print the oss request id for each error s3
request (#32491)
89048051c0d is described below
commit 89048051c0df2f4bd9ae320c09183098a6ff5131
Author: yujun <[email protected]>
AuthorDate: Wed Mar 20 23:36:39 2024 +0800
[enhance](S3) Print the oss request id for each error s3 request (#32491)
---
docker/runtime/doris-compose/cluster.py | 31 ++++++++++++++++++++++++++-----
docker/runtime/doris-compose/command.py | 11 +++++++++++
2 files changed, 37 insertions(+), 5 deletions(-)
diff --git a/docker/runtime/doris-compose/cluster.py
b/docker/runtime/doris-compose/cluster.py
index 660a243f5c4..cda6c3d845d 100644
--- a/docker/runtime/doris-compose/cluster.py
+++ b/docker/runtime/doris-compose/cluster.py
@@ -425,6 +425,9 @@ class BE(Node):
"meta_service_endpoint = {}".format(
self.cluster.get_meta_server_addr()),
'tmp_file_dirs = [
{"path":"./storage/tmp","max_cache_bytes":10240000,"
"max_upload_bytes":10240000}]',
+ 'enable_file_cache = true',
+ 'file_cache_path = [ {{"path": "{}/storage/file_cache",
"total_size":53687091200, "query_limit": 10737418240}}]'
+ .format(self.docker_home_dir()),
]
return cfg
@@ -464,6 +467,8 @@ class BE(Node):
for dir in dirs:
os.makedirs(dir, exist_ok=True)
+ os.makedirs(path + "/storage/file_cache", exist_ok=True)
+
with open("{}/conf/{}".format(path, self.conf_file_name()), "a") as f:
storage_root_path = ";".join(dir_descs) if dir_descs else '""'
f.write("\nstorage_root_path = {}\n".format(storage_root_path))
@@ -510,6 +515,12 @@ class CLOUD(Node):
class MS(CLOUD):
+ def get_add_init_config(self):
+ cfg = super().get_add_init_config()
+ if self.cluster.ms_config:
+ cfg += self.cluster.ms_config
+ return cfg
+
def entrypoint(self):
return [
"bash",
@@ -529,6 +540,12 @@ class MS(CLOUD):
class RECYCLE(CLOUD):
+ def get_add_init_config(self):
+ cfg = super().get_add_init_config()
+ if self.cluster.recycle_config:
+ cfg += self.cluster.recycle_config
+ return cfg
+
def entrypoint(self):
return [
"bash",
@@ -573,13 +590,16 @@ class FDB(Node):
class Cluster(object):
def __init__(self, name, subnet, image, is_cloud, fe_config, be_config,
- be_disks, be_cluster, coverage_dir, cloud_store_config):
+ ms_config, recycle_config, be_disks, be_cluster, coverage_dir,
+ cloud_store_config):
self.name = name
self.subnet = subnet
self.image = image
self.is_cloud = is_cloud
self.fe_config = fe_config
self.be_config = be_config
+ self.ms_config = ms_config
+ self.recycle_config = recycle_config
self.be_disks = be_disks
self.be_cluster = be_cluster
self.coverage_dir = coverage_dir
@@ -590,14 +610,15 @@ class Cluster(object):
}
@staticmethod
- def new(name, image, is_cloud, fe_config, be_config, be_disks, be_cluster,
- coverage_dir, cloud_store_config):
+ def new(name, image, is_cloud, fe_config, be_config, ms_config,
+ recycle_config, be_disks, be_cluster, coverage_dir,
+ cloud_store_config):
os.makedirs(LOCAL_DORIS_PATH, exist_ok=True)
with filelock.FileLock(os.path.join(LOCAL_DORIS_PATH, "lock")):
subnet = gen_subnet_prefix16()
cluster = Cluster(name, subnet, image, is_cloud, fe_config,
- be_config, be_disks, be_cluster, coverage_dir,
- cloud_store_config)
+ be_config, ms_config, recycle_config, be_disks,
+ be_cluster, coverage_dir, cloud_store_config)
os.makedirs(cluster.get_path(), exist_ok=True)
os.makedirs(get_status_path(name), exist_ok=True)
cluster._save_meta()
diff --git a/docker/runtime/doris-compose/command.py
b/docker/runtime/doris-compose/command.py
index 31c212eec8a..aa98e932872 100644
--- a/docker/runtime/doris-compose/command.py
+++ b/docker/runtime/doris-compose/command.py
@@ -235,6 +235,16 @@ class UpCommand(Command):
type=str,
help="Specify be configs for be.conf. "\
"Example: --be-config
\"enable_debug_points = true\" \"enable_auth = true\".")
+ group1.add_argument("--ms-config",
+ nargs="*",
+ type=str,
+ help="Specify ms configs for doris_cloud.conf. "\
+ "Example: --ms-config \"log_level =
warn\".")
+ group1.add_argument("--recycle-config",
+ nargs="*",
+ type=str,
+ help="Specify recycle configs for
doris_cloud.conf. "\
+ "Example: --recycle-config \"log_level =
warn\".")
group1.add_argument("--be-disks",
nargs="*",
default=["HDD=1"],
@@ -358,6 +368,7 @@ class UpCommand(Command):
cluster = CLUSTER.Cluster.new(args.NAME, args.IMAGE, args.cloud,
args.fe_config, args.be_config,
+ args.ms_config, args.recycle_config,
args.be_disks, args.be_cluster,
args.coverage_dir,
cloud_store_config)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
