date:20160229

kylin git commit: KYLIN-1458: Checking the consistency of cube segment host with the environment after cube migration

2016-02-29 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc fbbe50d78 -> ad8d71538


KYLIN-1458: Checking the consistency of cube segment host with the environment 
after cube migration

Signed-off-by: Hongbin Ma 

KYLIN-1458: Add the auto fix if inconsistency exists after cube migration

Signed-off-by: Hongbin Ma 

KYLIN-1458: rename parameters for avoiding misleading

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ad8d7153
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ad8d7153
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ad8d7153

Branch: refs/heads/2.0-rc
Commit: ad8d715389cbd9fd15202bc8507c8534a903cedc
Parents: fbbe50d
Author: yangzhong 
Authored: Mon Feb 29 09:37:06 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Mar 1 13:39:32 2016 +0800

--
 .../storage/hbase/util/CubeMigrationCLI.java|   7 +
 .../hbase/util/CubeMigrationCheckCLI.java   | 195 +++
 2 files changed, 202 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ad8d7153/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 57f362d..53cffe5 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -143,6 +143,8 @@ public class CubeMigrationCLI {
 } else {
 showOpts();
 }
+
+checkMigrationSuccess(dstConfig, cubeName, true);
 }
 
 public static void moveCube(String srcCfgUri, String dstCfgUri, String 
cubeName, String projectName, String copyAcl, String purgeAndDisable, String 
overwriteIfExists, String realExecute) throws IOException, InterruptedException 
{
@@ -150,6 +152,11 @@ public class CubeMigrationCLI {
 moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), 
KylinConfig.createInstanceFromUri(dstCfgUri), cubeName, projectName, copyAcl, 
purgeAndDisable, overwriteIfExists, realExecute);
 }
 
+public static void checkMigrationSuccess(KylinConfig kylinConfig, String 
cubeName, Boolean ifFix) throws IOException{
+CubeMigrationCheckCLI checkCLI = new 
CubeMigrationCheckCLI(kylinConfig, ifFix);
+checkCLI.execute(cubeName);
+}
+
 private static String checkAndGetHbaseUrl() {
 String srcMetadataUrl = srcConfig.getMetadataUrl();
 String dstMetadataUrl = dstConfig.getMetadataUrl();

http://git-wip-us.apache.org/repos/asf/kylin/blob/ad8d7153/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
new file mode 100644
index 000..2762561
--- /dev/null
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
@@ -0,0 +1,195 @@
+package org.apache.kylin.storage.hbase.util;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * 
+ * This tool serves for the purpose of
+ * checking the "KYLIN_HOST" property to be consistent with the dst's 
MetadataUrlPrefix
+ * for all of cube segments' corresponding HTables after migrating a cube
+ * 
+ */
+public class CubeMigrationCheckCLI {
+
+private static final Logger logger = 
LoggerFactory.getLogger(CubeMigrationCheckCLI.class);
+
+private static final Option OPTION_FIX =

kylin git commit: KYLIN-1458: Checking the consistency of cube segment host with the environment after cube migration

2016-02-29 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 5474fe4e8 -> ca5979554


KYLIN-1458: Checking the consistency of cube segment host with the environment 
after cube migration

Signed-off-by: Hongbin Ma 

KYLIN-1458: Add the auto fix if inconsistency exists after cube migration

Signed-off-by: Hongbin Ma 

KYLIN-1458: rename parameters for avoiding misleading

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ca597955
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ca597955
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ca597955

Branch: refs/heads/2.x-staging
Commit: ca597955477650a388f882b968e25499b02a4bf0
Parents: 5474fe4
Author: yangzhong 
Authored: Mon Feb 29 09:37:06 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Mar 1 13:38:02 2016 +0800

--
 .../storage/hbase/util/CubeMigrationCLI.java|   7 +
 .../hbase/util/CubeMigrationCheckCLI.java   | 195 +++
 2 files changed, 202 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ca597955/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index caf79b2..09aab48 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -141,6 +141,8 @@ public class CubeMigrationCLI {
 } else {
 showOpts();
 }
+
+checkMigrationSuccess(dstConfig, cubeName, true);
 }
 
 public static void moveCube(String srcCfgUri, String dstCfgUri, String 
cubeName, String projectName, String copyAcl, String purgeAndDisable, String 
overwriteIfExists, String realExecute) throws IOException, InterruptedException 
{
@@ -148,6 +150,11 @@ public class CubeMigrationCLI {
 moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), 
KylinConfig.createInstanceFromUri(dstCfgUri), cubeName, projectName, copyAcl, 
purgeAndDisable, overwriteIfExists, realExecute);
 }
 
+public static void checkMigrationSuccess(KylinConfig kylinConfig, String 
cubeName, Boolean ifFix) throws IOException{
+CubeMigrationCheckCLI checkCLI = new 
CubeMigrationCheckCLI(kylinConfig, ifFix);
+checkCLI.execute(cubeName);
+}
+
 private static String checkAndGetHbaseUrl() {
 String srcMetadataUrl = srcConfig.getMetadataUrl();
 String dstMetadataUrl = dstConfig.getMetadataUrl();

http://git-wip-us.apache.org/repos/asf/kylin/blob/ca597955/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
new file mode 100644
index 000..2762561
--- /dev/null
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
@@ -0,0 +1,195 @@
+package org.apache.kylin.storage.hbase.util;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * 
+ * This tool serves for the purpose of
+ * checking the "KYLIN_HOST" property to be consistent with the dst's 
MetadataUrlPrefix
+ * for all of cube segments' corresponding HTables after migrating a cube
+ * 
+ */
+public class CubeMigrationCheckCLI {
+
+private static final Logger logger = 
LoggerFactory.getLogger(CubeMigrationCheckCLI.class);
+
+private static final Option OPTION_FIX =

kylin git commit: KYLIN-579 Fix some minor issue on table controller and testcase

2016-02-29 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/1.x-staging 9eaf93544 -> a14da14ea


KYLIN-579 Fix some minor issue on table controller and testcase


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a14da14e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a14da14e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a14da14e

Branch: refs/heads/1.x-staging
Commit: a14da14ea9ebd647ce9605a896e9014e1b72298d
Parents: 9eaf935
Author: lidongsjtu 
Authored: Tue Mar 1 00:02:08 2016 +0800
Committer: lidongsjtu 
Committed: Tue Mar 1 10:17:19 2016 +0800

--
 cube/src/main/java/org/apache/kylin/cube/CubeManager.java   | 9 +
 .../apache/kylin/rest/controller/TableControllerTest.java   | 2 +-
 2 files changed, 6 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a14da14e/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 6fb907f..8a1a32b 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -170,7 +170,7 @@ public class CubeManager implements IRealizationProvider {
 
 public boolean isTableInCube(String tableName, String projectName) {
 ProjectManager projectManager = ProjectManager.getInstance(config);
-CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
+CubeManager cubeManager = CubeManager.getInstance(config);
 ProjectInstance projectInstance = 
projectManager.getProject(projectName);
 if (projectInstance == null) {
 throw new IllegalStateException("Cannot find project '" + 
projectName + "' in project manager");
@@ -178,11 +178,12 @@ public class CubeManager implements IRealizationProvider {
 
for (RealizationEntry projectDataModel : 
projectInstance.getRealizationEntries()) {
if (projectDataModel.getType() == RealizationType.CUBE) 
{
-   CubeDesc cubeDesc = 
cubeDescManager.getCubeDesc(projectDataModel.getRealization());
-if (cubeDesc == null) {
-throw new IllegalStateException("Cannot find cube '" + 
projectDataModel.getRealization() + "' in cubeDesc manager");
+CubeInstance cubeInstance = 
cubeManager.getCube(projectDataModel.getRealization());
+if (cubeInstance == null) {
+throw new IllegalStateException("Cannot find cube '" + 
projectDataModel.getRealization() + "' in cube manager.");
 }
 
+CubeDesc cubeDesc = cubeInstance.getDescriptor();
if 
(cubeDesc.getModel().getAllTables().contains(tableName.toUpperCase())) {
return true;
}

http://git-wip-us.apache.org/repos/asf/kylin/blob/a14da14e/server/src/test/java/org/apache/kylin/rest/controller/TableControllerTest.java
--
diff --git 
a/server/src/test/java/org/apache/kylin/rest/controller/TableControllerTest.java
 
b/server/src/test/java/org/apache/kylin/rest/controller/TableControllerTest.java
index 4a48285..e04a30e 100644
--- 
a/server/src/test/java/org/apache/kylin/rest/controller/TableControllerTest.java
+++ 
b/server/src/test/java/org/apache/kylin/rest/controller/TableControllerTest.java
@@ -74,6 +74,6 @@ public class TableControllerTest extends ServiceTestBase {
 Assert.assertTrue(loadResult.get("result.loaded").length == 2);
 
 loadResult = 
tableController.unLoadHiveTables("TEST_CATEGORY_GROUPINGS","default");
-
Assert.assertTrue(loadResult.get("result.unload.success")[0].equals("TEST_CATEGORY_GROUPINGS"));
+
Assert.assertTrue(loadResult.get("result.unload.fail")[0].equals("TEST_CATEGORY_GROUPINGS"));
 }
 }

kylin git commit: minor, fix issue that some job scheduler tests always fail in mvn test -fae

2016-02-29 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/1.x-staging 0917d485c -> 9eaf93544


minor, fix issue that some job scheduler tests always fail in mvn test -fae


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9eaf9354
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9eaf9354
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9eaf9354

Branch: refs/heads/1.x-staging
Commit: 9eaf935449d304ae58ff58f0ccb51c9d970b236b
Parents: 0917d48
Author: lidongsjtu 
Authored: Mon Feb 29 20:31:38 2016 +0800
Committer: lidongsjtu 
Committed: Mon Feb 29 20:31:38 2016 +0800

--
 .../job/impl/threadpool/BaseSchedulerTest.java  | 22 +---
 1 file changed, 15 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9eaf9354/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
--
diff --git 
a/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java 
b/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
index f3f4272..cb9535d 100644
--- 
a/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
+++ 
b/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
@@ -22,8 +22,8 @@ import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
 
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.lock.ZookeeperJobLock;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.common.lock.JobLock;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import org.apache.kylin.job.constant.ExecutableConstants;
 import org.apache.kylin.job.engine.JobEngineConfig;
 import org.apache.kylin.job.execution.AbstractExecutable;
@@ -32,10 +32,7 @@ import org.apache.kylin.job.manager.ExecutableManager;
 import org.junit.After;
 import org.junit.Before;
 
-/**
- * Created by qianzhou on 12/26/14.
- */
-public abstract class BaseSchedulerTest extends HBaseMetadataTestCase {
+public abstract class BaseSchedulerTest extends LocalFileMetadataTestCase {
 
 private DefaultScheduler scheduler;
 
@@ -88,7 +85,18 @@ public abstract class BaseSchedulerTest extends 
HBaseMetadataTestCase {
 
setFinalStatic(ExecutableConstants.class.getField("DEFAULT_SCHEDULER_INTERVAL_SECONDS"),
 10);
 jobService = 
ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv());
 scheduler = DefaultScheduler.getInstance();
-scheduler.init(new JobEngineConfig(KylinConfig.getInstanceFromEnv()), 
new ZookeeperJobLock());
+scheduler.init(new JobEngineConfig(KylinConfig.getInstanceFromEnv()), 
new JobLock() {
+@Override
+public boolean lock() {
+return true;
+}
+
+@Override
+public void unlock() {
+return;
+}
+});
+
 if (!scheduler.hasStarted()) {
 throw new RuntimeException("scheduler has not been started");
 }

kylin git commit: KYLIN-1458: Checking the consistency of cube segment host with the environment after cube migration

kylin git commit: KYLIN-1458: Checking the consistency of cube segment host with the environment after cube migration

kylin git commit: KYLIN-579 Fix some minor issue on table controller and testcase

kylin git commit: minor, fix issue that some job scheduler tests always fail in mvn test -fae

4 matches

Site Navigation

Mail list logo

Footer information