Repository: kylin Updated Branches: refs/heads/master 53aafa97e -> c38def7b5
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java b/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java index 7709760..8e17645 100644 --- a/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java +++ b/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java @@ -46,13 +46,10 @@ import com.google.common.collect.Sets; public class MetadataCleanupJob extends AbstractHadoopJob { @SuppressWarnings("static-access") - private static final Option OPTION_DELETE = OptionBuilder.withArgName("delete").hasArg().isRequired(false) - .withDescription("Delete the unused metadata").create("delete"); + private static final Option OPTION_DELETE = OptionBuilder.withArgName("delete").hasArg().isRequired(false).withDescription("Delete the unused metadata").create("delete"); @SuppressWarnings("static-access") - private static final Option OPTION_THRESHOLD_FOR_JOB = OptionBuilder.withArgName("jobThreshold").hasArg() - .isRequired(false).withDescription("Specify how many days of job metadata keeping. Default 30 days") - .create("jobThreshold"); + private static final Option OPTION_THRESHOLD_FOR_JOB = OptionBuilder.withArgName("jobThreshold").hasArg().isRequired(false).withDescription("Specify how many days of job metadata keeping. Default 30 days").create("jobThreshold"); protected static final Logger logger = LoggerFactory.getLogger(MetadataCleanupJob.class); @@ -61,7 +58,7 @@ public class MetadataCleanupJob extends AbstractHadoopJob { private KylinConfig config = null; private static final long TIME_THREADSHOLD = 1 * 3600 * 1000L; // 1 hour - private static final int DEFAULT_DAY_THREADSHOLD_FOR_JOB = 30; // 30 days + private static final int DEFAULT_DAY_THREADSHOLD_FOR_JOB = 30 ; // 30 days /* * (non-Javadoc) @@ -107,8 +104,7 @@ public class MetadataCleanupJob extends AbstractHadoopJob { List<String> toDeleteResource = Lists.newArrayList(); // two level resources, snapshot tables and cube statistics - for (String resourceRoot : new String[] { ResourceStore.SNAPSHOT_RESOURCE_ROOT, - ResourceStore.CUBE_STATISTICS_ROOT }) { + for (String resourceRoot : new String[] { ResourceStore.SNAPSHOT_RESOURCE_ROOT, ResourceStore.CUBE_STATISTICS_ROOT }) { NavigableSet<String> snapshotTables = getStore().listResources(resourceRoot); if (snapshotTables != null) { @@ -157,14 +153,10 @@ public class MetadataCleanupJob extends AbstractHadoopJob { for (ExecutablePO executable : allExecutable) { long lastModified = executable.getLastModified(); ExecutableOutputPO output = executableDao.getJobOutput(executable.getUuid()); - int jobThresholdDay = optionsHelper.hasOption(OPTION_THRESHOLD_FOR_JOB) - ? Integer.valueOf(optionsHelper.getOptionValue(OPTION_THRESHOLD_FOR_JOB)) - : DEFAULT_DAY_THREADSHOLD_FOR_JOB; + int jobThresholdDay = optionsHelper.hasOption(OPTION_THRESHOLD_FOR_JOB) ? Integer.valueOf(optionsHelper.getOptionValue(OPTION_THRESHOLD_FOR_JOB)) : DEFAULT_DAY_THREADSHOLD_FOR_JOB; long jobThresholdTime = jobThresholdDay * 24 * 3600 * 1000L; - if (System.currentTimeMillis() - lastModified > jobThresholdTime - && (ExecutableState.SUCCEED.toString().equals(output.getStatus()) - || ExecutableState.DISCARDED.toString().equals(output.getStatus()))) { + if (System.currentTimeMillis() - lastModified > jobThresholdTime && (ExecutableState.SUCCEED.toString().equals(output.getStatus()) || ExecutableState.DISCARDED.toString().equals(output.getStatus()))) { toDeleteResource.add(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + executable.getUuid()); toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + executable.getUuid()); @@ -175,8 +167,7 @@ public class MetadataCleanupJob extends AbstractHadoopJob { } if (toDeleteResource.size() > 0) { - logger.info( - "The following resources have no reference or is too old, will be cleaned from metadata store: \n"); + logger.info("The following resources have no reference or is too old, will be cleaned from metadata store: \n"); for (String s : toDeleteResource) { logger.info(s); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java b/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java index b18f812..ca4c7e1 100644 --- a/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java +++ b/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java @@ -54,13 +54,10 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor { private static final Logger logger = LoggerFactory.getLogger(MrJobInfoExtractor.class); @SuppressWarnings("static-access") - private static final Option OPTION_INCLUDE_DETAILS = OptionBuilder.withArgName("includeTasks").hasArg() - .isRequired(false).withDescription("Specify whether to include mr task details to extract. Default true.") - .create("includeTasks"); + private static final Option OPTION_INCLUDE_DETAILS = OptionBuilder.withArgName("includeTasks").hasArg().isRequired(false).withDescription("Specify whether to include mr task details to extract. Default true.").create("includeTasks"); @SuppressWarnings("static-access") - private static final Option OPTION_MR_JOB_ID = OptionBuilder.withArgName("mrJobId").hasArg().isRequired(false) - .withDescription("Specify MR Job Id").create("mrJobId"); + private static final Option OPTION_MR_JOB_ID = OptionBuilder.withArgName("mrJobId").hasArg().isRequired(false).withDescription("Specify MR Job Id").create("mrJobId"); private static final int HTTP_RETRY = 3; @@ -95,13 +92,11 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor { logger.info("kylin.engine.mr.yarn-check-status-url" + " is not set, read from hadoop configuration"); Configuration conf = HadoopUtil.getCurrentConfiguration(); - String rmWebHost = HAUtil.getConfValueForRMInstance(YarnConfiguration.RM_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, conf); + String rmWebHost = HAUtil.getConfValueForRMInstance(YarnConfiguration.RM_WEBAPP_ADDRESS, YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, conf); if (HAUtil.isHAEnabled(conf)) { YarnConfiguration yarnConf = new YarnConfiguration(conf); String active = RMHAUtils.findActiveRMHAId(yarnConf); - rmWebHost = HAUtil.getConfValueForRMInstance(HAUtil.addSuffix(YarnConfiguration.RM_WEBAPP_ADDRESS, active), - YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, yarnConf); + rmWebHost = HAUtil.getConfValueForRMInstance(HAUtil.addSuffix(YarnConfiguration.RM_WEBAPP_ADDRESS, active), YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, yarnConf); } if (StringUtils.isEmpty(rmWebHost)) { return; @@ -112,8 +107,7 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor { Matcher m = pattern.matcher(rmWebHost); Preconditions.checkArgument(m.matches(), "Yarn master URL not found."); yarnMasterUrlBase = rmWebHost; - jobHistoryUrlBase = m.group(1) - + HAUtil.getConfValueForRMInstance("mapreduce.jobhistory.webapp.address", m.group(2) + ":19888", conf); + jobHistoryUrlBase = m.group(1) + HAUtil.getConfValueForRMInstance("mapreduce.jobhistory.webapp.address", m.group(2) + ":19888", conf); } private String getHttpResponse(String url) { @@ -137,8 +131,7 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor { return msg; } - private void extractTaskDetail(String taskId, String user, File exportDir, String taskUrl, String urlBase) - throws IOException { + private void extractTaskDetail(String taskId, String user, File exportDir, String taskUrl, String urlBase) throws IOException { try { if (StringUtils.isEmpty(taskId)) { return; @@ -152,8 +145,7 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor { JsonNode taskAttempt = new ObjectMapper().readTree(taskInfo).path("task").path("successfulAttempt"); String succAttemptId = taskAttempt.textValue(); - String attemptInfo = saveHttpResponseQuietly(new File(destDir, "task_attempts.json"), - taskUrlBase + "/attempts/" + succAttemptId); + String attemptInfo = saveHttpResponseQuietly(new File(destDir, "task_attempts.json"), taskUrlBase + "/attempts/" + succAttemptId); JsonNode attemptAttempt = new ObjectMapper().readTree(attemptInfo).path("taskAttempt"); String containerId = attemptAttempt.get("assignedContainerId").textValue(); String nodeId = nodeInfoMap.get(attemptAttempt.get("nodeHttpAddress").textValue()); @@ -162,8 +154,7 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor { saveHttpResponseQuietly(new File(destDir, "task_counters.json"), taskUrlBase + "/counters"); // save task logs - String logUrl = urlBase + "/jobhistory/logs/" + nodeId + "/" + containerId + "/" + succAttemptId + "/" - + user + "/syslog/?start=0"; + String logUrl = urlBase + "/jobhistory/logs/" + nodeId + "/" + containerId + "/" + succAttemptId + "/" + user + "/syslog/?start=0"; logger.debug("Fetch task log from url: " + logUrl); saveHttpResponseQuietly(new File(destDir, "task_log.txt"), logUrl); @@ -190,8 +181,7 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor { @Override protected void executeExtract(OptionsHelper optionsHelper, File exportDir) throws Exception { try { - boolean includeTaskDetails = optionsHelper.hasOption(OPTION_INCLUDE_DETAILS) - ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_DETAILS)) : true; + boolean includeTaskDetails = optionsHelper.hasOption(OPTION_INCLUDE_DETAILS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_DETAILS)) : true; String mrJobId = optionsHelper.getOptionValue(OPTION_MR_JOB_ID); extractRestCheckUrl(); @@ -359,4 +349,4 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor { logger.warn("Failed to get mr tasks rest response.", e); } } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java b/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java index 70c3be2..2e357da 100644 --- a/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java +++ b/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java @@ -19,18 +19,17 @@ package org.apache.kylin.tool.util; +import com.google.common.collect.Maps; +import org.apache.commons.lang.StringUtils; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.persistence.ResourceStore; + import java.io.File; import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Map; -import org.apache.commons.lang.StringUtils; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.ResourceStore; - -import com.google.common.collect.Maps; - public class ToolUtil { public static String getConfFolder() { final String CONF = "conf"; http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java ---------------------------------------------------------------------- diff --git a/tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java b/tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java index 063a145..ff12084 100644 --- a/tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java +++ b/tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java @@ -58,15 +58,13 @@ public class CubeMetaIngesterTest extends LocalFileMetadataTestCase { @Test public void testHappyIngest() { - String srcPath = Thread.currentThread().getContextClassLoader().getResource("cloned_cube_and_model.zip") - .getPath(); + String srcPath = Thread.currentThread().getContextClassLoader().getResource("cloned_cube_and_model.zip").getPath(); CubeMetaIngester.main(new String[] { "-project", "default", "-srcPath", srcPath }); ProjectInstance project = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProject("default"); Assert.assertEquals(1, Collections.frequency(project.getTables(), "DEFAULT.TEST_KYLIN_FACT")); Assert.assertTrue(project.getModels().contains("cloned_model")); - Assert.assertTrue( - project.getRealizationEntries().contains(RealizationEntry.create(RealizationType.CUBE, "cloned_cube"))); + Assert.assertTrue(project.getRealizationEntries().contains(RealizationEntry.create(RealizationType.CUBE, "cloned_cube"))); MetadataManager.clearCache(); CubeDescManager.clearCache(); @@ -83,8 +81,7 @@ public class CubeMetaIngesterTest extends LocalFileMetadataTestCase { ProjectInstance project = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProject("default"); Assert.assertEquals(1, Collections.frequency(project.getTables(), "SSB.CUSTOMER")); Assert.assertTrue(project.getModels().contains("benchmark_model")); - Assert.assertTrue(project.getRealizationEntries() - .contains(RealizationEntry.create(RealizationType.CUBE, "benchmark_cube"))); + Assert.assertTrue(project.getRealizationEntries().contains(RealizationEntry.create(RealizationType.CUBE, "benchmark_cube"))); MetadataManager.clearCache(); CubeDescManager.clearCache(); @@ -102,8 +99,7 @@ public class CubeMetaIngesterTest extends LocalFileMetadataTestCase { @Override public boolean matches(Object item) { if (item instanceof IllegalStateException) { - if (((IllegalStateException) item).getMessage() - .equals("Already exist a model called test_kylin_inner_join_model_desc")) { + if (((IllegalStateException) item).getMessage().equals("Already exist a model called test_kylin_inner_join_model_desc")) { return true; } } @@ -129,4 +125,4 @@ public class CubeMetaIngesterTest extends LocalFileMetadataTestCase { CubeMetaIngester.main(new String[] { "-project", "Xdefault", "-srcPath", srcPath }); } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java ---------------------------------------------------------------------- diff --git a/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java b/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java index 68f9ff8..f4ed595 100644 --- a/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java +++ b/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java @@ -50,8 +50,7 @@ public class HybridCubeCLITest extends LocalFileMetadataTestCase { public void test1Create() throws IOException { HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv()); Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid")); - HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", - "ssb_cube1,ssb_cube2", "-action", "create" }); + HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" }); HybridInstance hybridInstance = hybridManager.getHybridInstance("ssb_hybrid"); Assert.assertNotNull(hybridInstance); @@ -63,15 +62,13 @@ public class HybridCubeCLITest extends LocalFileMetadataTestCase { public void test2Update() throws IOException { HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv()); Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid")); - HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", - "ssb_cube1,ssb_cube2", "-action", "create" }); + HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" }); HybridInstance hybridInstance = hybridManager.getHybridInstance("ssb_hybrid"); Assert.assertNotNull(hybridManager.getHybridInstance("ssb_hybrid")); Assert.assertEquals("ssb_hybrid", hybridInstance.getName()); Assert.assertEquals(2, hybridInstance.getRealizationEntries().size()); - HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", - "ssb_cube1,ssb_cube2,ssb_cube3", "-action", "update" }); + HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2,ssb_cube3", "-action", "update" }); hybridInstance = hybridManager.getHybridInstance("ssb_hybrid"); Assert.assertNotNull(hybridInstance); @@ -83,16 +80,13 @@ public class HybridCubeCLITest extends LocalFileMetadataTestCase { public void test3Delete() throws IOException { HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv()); Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid")); - HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", - "ssb_cube1,ssb_cube2", "-action", "create" }); + HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" }); Assert.assertNotNull(hybridManager.getHybridInstance("ssb_hybrid")); - HybridCubeCLI.main( - new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-action", "delete" }); + HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-action", "delete" }); HybridInstance hybridInstance = hybridManager.getHybridInstance("ssb_hybrid"); Assert.assertNull(hybridInstance); - Assert.assertEquals(0, ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()) - .findProjects(RealizationType.HYBRID, "ssb_hybrid").size()); + Assert.assertEquals(0, ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).findProjects(RealizationType.HYBRID, "ssb_hybrid").size()); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java ---------------------------------------------------------------------- diff --git a/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java b/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java index 61a4b52..7d1e248 100644 --- a/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java +++ b/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java @@ -55,9 +55,7 @@ public class KylinConfigCLITest extends LocalFileMetadataTestCase { KylinConfigCLI.main(new String[] { "kylin.cube.engine." }); String val = FileUtils.readFileToString(f, Charset.defaultCharset()).trim(); - assertEquals( - "2=org.apache.kylin.engine.mr.MRBatchCubingEngine2\n0=org.apache.kylin.engine.mr.MRBatchCubingEngine", - val); + assertEquals("2=org.apache.kylin.engine.mr.MRBatchCubingEngine2\n0=org.apache.kylin.engine.mr.MRBatchCubingEngine", val); FileUtils.forceDelete(f); System.setOut(o);
