[17/50] [abbrv] kylin git commit: KYLIN-1458: Checking the consistency of cube segment host with the environment after cube migration

2016-03-09 Thread liyang 
KYLIN-1458: Checking the consistency of cube segment host with the environment 
after cube migration

Signed-off-by: Hongbin Ma 

KYLIN-1458: Add the auto fix if inconsistency exists after cube migration

Signed-off-by: Hongbin Ma 

KYLIN-1458: rename parameters for avoiding misleading

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ca597955
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ca597955
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ca597955

Branch: refs/heads/master
Commit: ca597955477650a388f882b968e25499b02a4bf0
Parents: 5474fe4
Author: yangzhong 
Authored: Mon Feb 29 09:37:06 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Mar 1 13:38:02 2016 +0800

--
 .../storage/hbase/util/CubeMigrationCLI.java|   7 +
 .../hbase/util/CubeMigrationCheckCLI.java   | 195 +++
 2 files changed, 202 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ca597955/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index caf79b2..09aab48 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -141,6 +141,8 @@ public class CubeMigrationCLI {
 } else {
 showOpts();
 }
+
+checkMigrationSuccess(dstConfig, cubeName, true);
 }
 
 public static void moveCube(String srcCfgUri, String dstCfgUri, String 
cubeName, String projectName, String copyAcl, String purgeAndDisable, String 
overwriteIfExists, String realExecute) throws IOException, InterruptedException 
{
@@ -148,6 +150,11 @@ public class CubeMigrationCLI {
 moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), 
KylinConfig.createInstanceFromUri(dstCfgUri), cubeName, projectName, copyAcl, 
purgeAndDisable, overwriteIfExists, realExecute);
 }
 
+public static void checkMigrationSuccess(KylinConfig kylinConfig, String 
cubeName, Boolean ifFix) throws IOException{
+CubeMigrationCheckCLI checkCLI = new 
CubeMigrationCheckCLI(kylinConfig, ifFix);
+checkCLI.execute(cubeName);
+}
+
 private static String checkAndGetHbaseUrl() {
 String srcMetadataUrl = srcConfig.getMetadataUrl();
 String dstMetadataUrl = dstConfig.getMetadataUrl();

http://git-wip-us.apache.org/repos/asf/kylin/blob/ca597955/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
new file mode 100644
index 000..2762561
--- /dev/null
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
@@ -0,0 +1,195 @@
+package org.apache.kylin.storage.hbase.util;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * 
+ * This tool serves for the purpose of
+ * checking the "KYLIN_HOST" property to be consistent with the dst's 
MetadataUrlPrefix
+ * for all of cube segments' corresponding HTables after migrating a cube
+ * 
+ */
+public class CubeMigrationCheckCLI {
+
+private static final Logger logger = 
LoggerFactory.getLogger(CubeMigrationCheckCLI.class);
+
+private static final Option OPTION_FIX = 
OptionBuilder.withArgName("fix").hasArg().isRequired(false).withDescription("Fix
 the inconsistent cube

[08/43] kylin git commit: KYLIN-1458: Checking the consistency of cube segment host with the environment after cube migration

2016-03-03 Thread shaofengshi 
KYLIN-1458: Checking the consistency of cube segment host with the environment 
after cube migration

Signed-off-by: Hongbin Ma 

KYLIN-1458: Add the auto fix if inconsistency exists after cube migration

Signed-off-by: Hongbin Ma 

KYLIN-1458: rename parameters for avoiding misleading

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ca597955
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ca597955
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ca597955

Branch: refs/heads/helix-rebase
Commit: ca597955477650a388f882b968e25499b02a4bf0
Parents: 5474fe4
Author: yangzhong 
Authored: Mon Feb 29 09:37:06 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Mar 1 13:38:02 2016 +0800

--
 .../storage/hbase/util/CubeMigrationCLI.java|   7 +
 .../hbase/util/CubeMigrationCheckCLI.java   | 195 +++
 2 files changed, 202 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ca597955/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index caf79b2..09aab48 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -141,6 +141,8 @@ public class CubeMigrationCLI {
 } else {
 showOpts();
 }
+
+checkMigrationSuccess(dstConfig, cubeName, true);
 }
 
 public static void moveCube(String srcCfgUri, String dstCfgUri, String 
cubeName, String projectName, String copyAcl, String purgeAndDisable, String 
overwriteIfExists, String realExecute) throws IOException, InterruptedException 
{
@@ -148,6 +150,11 @@ public class CubeMigrationCLI {
 moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), 
KylinConfig.createInstanceFromUri(dstCfgUri), cubeName, projectName, copyAcl, 
purgeAndDisable, overwriteIfExists, realExecute);
 }
 
+public static void checkMigrationSuccess(KylinConfig kylinConfig, String 
cubeName, Boolean ifFix) throws IOException{
+CubeMigrationCheckCLI checkCLI = new 
CubeMigrationCheckCLI(kylinConfig, ifFix);
+checkCLI.execute(cubeName);
+}
+
 private static String checkAndGetHbaseUrl() {
 String srcMetadataUrl = srcConfig.getMetadataUrl();
 String dstMetadataUrl = dstConfig.getMetadataUrl();

http://git-wip-us.apache.org/repos/asf/kylin/blob/ca597955/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
new file mode 100644
index 000..2762561
--- /dev/null
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
@@ -0,0 +1,195 @@
+package org.apache.kylin.storage.hbase.util;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * 
+ * This tool serves for the purpose of
+ * checking the "KYLIN_HOST" property to be consistent with the dst's 
MetadataUrlPrefix
+ * for all of cube segments' corresponding HTables after migrating a cube
+ * 
+ */
+public class CubeMigrationCheckCLI {
+
+private static final Logger logger = 
LoggerFactory.getLogger(CubeMigrationCheckCLI.class);
+
+private static final Option OPTION_FIX = 
OptionBuilder.withArgName("fix").hasArg().isRequired(false).withDescription("Fix
 the inconsistent cube

kylin git commit: KYLIN-1458: Checking the consistency of cube segment host with the environment after cube migration

2016-02-29 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/2.0-rc fbbe50d78 -> ad8d71538


KYLIN-1458: Checking the consistency of cube segment host with the environment 
after cube migration

Signed-off-by: Hongbin Ma 

KYLIN-1458: Add the auto fix if inconsistency exists after cube migration

Signed-off-by: Hongbin Ma 

KYLIN-1458: rename parameters for avoiding misleading

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ad8d7153
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ad8d7153
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ad8d7153

Branch: refs/heads/2.0-rc
Commit: ad8d715389cbd9fd15202bc8507c8534a903cedc
Parents: fbbe50d
Author: yangzhong 
Authored: Mon Feb 29 09:37:06 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Mar 1 13:39:32 2016 +0800

--
 .../storage/hbase/util/CubeMigrationCLI.java|   7 +
 .../hbase/util/CubeMigrationCheckCLI.java   | 195 +++
 2 files changed, 202 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ad8d7153/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 57f362d..53cffe5 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -143,6 +143,8 @@ public class CubeMigrationCLI {
 } else {
 showOpts();
 }
+
+checkMigrationSuccess(dstConfig, cubeName, true);
 }
 
 public static void moveCube(String srcCfgUri, String dstCfgUri, String 
cubeName, String projectName, String copyAcl, String purgeAndDisable, String 
overwriteIfExists, String realExecute) throws IOException, InterruptedException 
{
@@ -150,6 +152,11 @@ public class CubeMigrationCLI {
 moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), 
KylinConfig.createInstanceFromUri(dstCfgUri), cubeName, projectName, copyAcl, 
purgeAndDisable, overwriteIfExists, realExecute);
 }
 
+public static void checkMigrationSuccess(KylinConfig kylinConfig, String 
cubeName, Boolean ifFix) throws IOException{
+CubeMigrationCheckCLI checkCLI = new 
CubeMigrationCheckCLI(kylinConfig, ifFix);
+checkCLI.execute(cubeName);
+}
+
 private static String checkAndGetHbaseUrl() {
 String srcMetadataUrl = srcConfig.getMetadataUrl();
 String dstMetadataUrl = dstConfig.getMetadataUrl();

http://git-wip-us.apache.org/repos/asf/kylin/blob/ad8d7153/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
new file mode 100644
index 000..2762561
--- /dev/null
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
@@ -0,0 +1,195 @@
+package org.apache.kylin.storage.hbase.util;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * 
+ * This tool serves for the purpose of
+ * checking the "KYLIN_HOST" property to be consistent with the dst's 
MetadataUrlPrefix
+ * for all of cube segments' corresponding HTables after migrating a cube
+ * 
+ */
+public class CubeMigrationCheckCLI {
+
+private static final Logger logger = 
LoggerFactory.getLogger(CubeMigrationCheckCLI.class);
+
+private static final Option OPTION_FIX =

kylin git commit: KYLIN-1458: Checking the consistency of cube segment host with the environment after cube migration

2016-02-29 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 5474fe4e8 -> ca5979554


KYLIN-1458: Checking the consistency of cube segment host with the environment 
after cube migration

Signed-off-by: Hongbin Ma 

KYLIN-1458: Add the auto fix if inconsistency exists after cube migration

Signed-off-by: Hongbin Ma 

KYLIN-1458: rename parameters for avoiding misleading

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ca597955
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ca597955
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ca597955

Branch: refs/heads/2.x-staging
Commit: ca597955477650a388f882b968e25499b02a4bf0
Parents: 5474fe4
Author: yangzhong 
Authored: Mon Feb 29 09:37:06 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Mar 1 13:38:02 2016 +0800

--
 .../storage/hbase/util/CubeMigrationCLI.java|   7 +
 .../hbase/util/CubeMigrationCheckCLI.java   | 195 +++
 2 files changed, 202 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ca597955/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index caf79b2..09aab48 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -141,6 +141,8 @@ public class CubeMigrationCLI {
 } else {
 showOpts();
 }
+
+checkMigrationSuccess(dstConfig, cubeName, true);
 }
 
 public static void moveCube(String srcCfgUri, String dstCfgUri, String 
cubeName, String projectName, String copyAcl, String purgeAndDisable, String 
overwriteIfExists, String realExecute) throws IOException, InterruptedException 
{
@@ -148,6 +150,11 @@ public class CubeMigrationCLI {
 moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), 
KylinConfig.createInstanceFromUri(dstCfgUri), cubeName, projectName, copyAcl, 
purgeAndDisable, overwriteIfExists, realExecute);
 }
 
+public static void checkMigrationSuccess(KylinConfig kylinConfig, String 
cubeName, Boolean ifFix) throws IOException{
+CubeMigrationCheckCLI checkCLI = new 
CubeMigrationCheckCLI(kylinConfig, ifFix);
+checkCLI.execute(cubeName);
+}
+
 private static String checkAndGetHbaseUrl() {
 String srcMetadataUrl = srcConfig.getMetadataUrl();
 String dstMetadataUrl = dstConfig.getMetadataUrl();

http://git-wip-us.apache.org/repos/asf/kylin/blob/ca597955/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
new file mode 100644
index 000..2762561
--- /dev/null
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
@@ -0,0 +1,195 @@
+package org.apache.kylin.storage.hbase.util;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * 
+ * This tool serves for the purpose of
+ * checking the "KYLIN_HOST" property to be consistent with the dst's 
MetadataUrlPrefix
+ * for all of cube segments' corresponding HTables after migrating a cube
+ * 
+ */
+public class CubeMigrationCheckCLI {
+
+private static final Logger logger = 
LoggerFactory.getLogger(CubeMigrationCheckCLI.class);
+
+private static final Option OPTION_FIX =