KYLIN-1761: Fetch part of metadata to local
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/87115444 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/87115444 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/87115444 Branch: refs/heads/1.4-rc Commit: 87115444731814f3fa1db1f70caa04e54b28b352 Parents: b220886 Author: kyotoYaho <nju_y...@apache.org> Authored: Fri Jun 3 19:41:10 2016 +0800 Committer: Zhong <yangzh...@lm-shc-16501214.corp.ebay.com> Committed: Fri Jun 3 19:59:48 2016 +0800 ---------------------------------------------------------------------- build/bin/metastore.sh | 14 ++++++++++++++ .../apache/kylin/common/persistence/ResourceTool.java | 10 ++++++++++ .../storage/hbase/steps/SandboxMetastoreCLI.java | 2 ++ 3 files changed, 26 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/87115444/build/bin/metastore.sh ---------------------------------------------------------------------- diff --git a/build/bin/metastore.sh b/build/bin/metastore.sh index 8908351..8d85661 100755 --- a/build/bin/metastore.sh +++ b/build/bin/metastore.sh @@ -40,6 +40,19 @@ then ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.common.persistence.ResourceTool download ${_file} echo "metadata store backed up to ${_file}" +elif [ "$1" == "fetch" ] +then + + _file=$2 + + _now=$(date +"%Y_%m_%d_%H_%M_%S") + _fileDst="${KYLIN_HOME}/meta_backups/meta_${_now}" + echo "Starting restoring $_fileDst" + mkdir -p $_fileDst + + ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.common.persistence.ResourceTool fetch $_fileDst $_file + echo "metadata store backed up to $_fileDst" + elif [ "$1" == "restore" ] then @@ -80,6 +93,7 @@ then else echo "usage: metastore.sh backup" + echo " metastore.sh fetch DATA" echo " metastore.sh reset" echo " metastore.sh restore PATH_TO_LOCAL_META" echo " metastore.sh list RESOURCE_PATH" http://git-wip-us.apache.org/repos/asf/kylin/blob/87115444/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java index 0975d57..187601a 100644 --- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java +++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java @@ -65,6 +65,9 @@ public class ResourceTool { case "download": copy(KylinConfig.getInstanceFromEnv(), KylinConfig.createInstanceFromUri(args[1])); break; + case "fetch": + copy(KylinConfig.getInstanceFromEnv(), KylinConfig.createInstanceFromUri(args[1]), args[2]); + break; case "upload": copy(KylinConfig.createInstanceFromUri(args[1]), KylinConfig.getInstanceFromEnv()); break; @@ -82,6 +85,13 @@ public class ResourceTool { System.out.println("" + result); } + public static void copy(KylinConfig srcConfig, KylinConfig dstConfig, String path) throws IOException { + ResourceStore src = ResourceStore.getStore(srcConfig); + ResourceStore dst = ResourceStore.getStore(dstConfig); + + copyR(src, dst, path); + } + public static void copy(KylinConfig srcConfig, KylinConfig dstConfig, List<String> paths) throws IOException { ResourceStore src = ResourceStore.getStore(srcConfig); ResourceStore dst = ResourceStore.getStore(dstConfig); http://git-wip-us.apache.org/repos/asf/kylin/blob/87115444/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java index 7b3b698..dbdb730 100644 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java +++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java @@ -54,6 +54,8 @@ public class SandboxMetastoreCLI { if ("download".equalsIgnoreCase(args[0])) { ResourceTool.main(new String[] { "download", args[1] }); + } else if ("fetch".equalsIgnoreCase(args[0])) { + ResourceTool.main(new String[] { "fetch", args[1], args[2]}); } else if ("upload".equalsIgnoreCase(args[0])) { ResourceTool.main(new String[] { "upload", args[1] }); } else {