KYLIN-1614 Add more env check to diag tool
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0308fa81 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0308fa81 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0308fa81 Branch: refs/heads/1.5.x-HBase1.x Commit: 0308fa81146fd0b0e100da0e5005a845cba70958 Parents: e47a648 Author: lidongsjtu <lid...@apache.org> Authored: Sat Apr 30 15:52:32 2016 +0800 Committer: lidongsjtu <lid...@apache.org> Committed: Sun May 1 14:14:18 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/KylinVersion.java | 1 + .../apache/kylin/rest/init/ClientInfoTask.java | 13 +++++----- .../kylin/tool/AbstractInfoExtractor.java | 19 ++++++++------ .../apache/kylin/tool/ClientEnvExtractor.java | 17 +++++++++--- .../org/apache/kylin/tool/DiagnosisInfoCLI.java | 27 ++++---------------- .../apache/kylin/tool/JobDiagnosisInfoCLI.java | 6 ++--- .../java/org/apache/kylin/tool/ToolUtil.java | 2 +- 7 files changed, 41 insertions(+), 44 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/0308fa81/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java index 03a0720..c015e62 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java @@ -126,6 +126,7 @@ public class KylinVersion { public static String getKylinClientInformation() { StringBuilder buf = new StringBuilder(); + buf.append("kylin.home: ").append(new File(KylinConfig.getKylinHome()).getAbsolutePath()).append("\n"); buf.append("kylin.version:").append(KylinVersion.getCurrentVersion()).append("\n"); buf.append("commit:").append(getGitCommitInfo()).append("\n"); buf.append("os.name:").append(System.getProperty("os.name")).append("\n"); http://git-wip-us.apache.org/repos/asf/kylin/blob/0308fa81/server/src/main/java/org/apache/kylin/rest/init/ClientInfoTask.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/init/ClientInfoTask.java b/server/src/main/java/org/apache/kylin/rest/init/ClientInfoTask.java index 0fc0e69..301a9df 100644 --- a/server/src/main/java/org/apache/kylin/rest/init/ClientInfoTask.java +++ b/server/src/main/java/org/apache/kylin/rest/init/ClientInfoTask.java @@ -18,13 +18,13 @@ package org.apache.kylin.rest.init; +import java.io.File; + +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.KylinVersion; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Created by dongli on 3/16/16. - */ public class ClientInfoTask extends InitialTask { private static final Logger logger = LoggerFactory.getLogger(ClientInfoTask.class); @@ -37,15 +37,14 @@ public class ClientInfoTask extends InitialTask { public static String getClientDetailInformation() { StringBuilder buf = new StringBuilder(); + buf.append("kylin.home: ").append(new File(KylinConfig.getKylinHome()).getAbsolutePath()).append("\n"); buf.append("kylin.version:").append(KylinVersion.getCurrentVersion()).append("\n"); - buf.append("commit:").append(KylinVersion.getGitCommitInfo()); + buf.append("commit:").append(KylinVersion.getGitCommitInfo()).append("\n"); buf.append("os.name:").append(System.getProperty("os.name")).append("\n"); buf.append("os.arch:").append(System.getProperty("os.arch")).append("\n"); buf.append("os.version:").append(System.getProperty("os.version")).append("\n"); - buf.append("java.class.path:").append(System.getProperty("java.class.path")).append("\n"); buf.append("java.version:").append(System.getProperty("java.version")).append("\n"); - buf.append("java.vendor:").append(System.getProperty("java.vendor")).append("\n"); - buf.append("java.library.path:").append(System.getProperty("java.library.path")); + buf.append("java.vendor:").append(System.getProperty("java.vendor")); return buf.toString(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/0308fa81/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java b/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java index 23a4559..4a5f31e 100644 --- a/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java +++ b/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java @@ -45,7 +45,7 @@ public abstract class AbstractInfoExtractor extends AbstractApplication { private static final Option OPTION_COMPRESS = OptionBuilder.withArgName("compress").hasArg().isRequired(false).withDescription("specify whether to compress the output with zip. Default true.").create("compress"); @SuppressWarnings("static-access") - private static final Option OPTION_QUIET = OptionBuilder.withArgName("quiet").hasArg().isRequired(false).withDescription("specify whether to print final result").create("quiet"); + private static final Option OPTION_SUBMODULE = OptionBuilder.withArgName("submodule").hasArg().isRequired(false).withDescription("specify whether this is a submodule of other CLI tool").create("submodule"); private static final String DEFAULT_PACKAGE_TYPE = "base"; @@ -58,7 +58,7 @@ public abstract class AbstractInfoExtractor extends AbstractApplication { options = new Options(); options.addOption(OPTION_DEST); options.addOption(OPTION_COMPRESS); - options.addOption(OPTION_QUIET); + options.addOption(OPTION_SUBMODULE); packageType = DEFAULT_PACKAGE_TYPE; } @@ -71,8 +71,8 @@ public abstract class AbstractInfoExtractor extends AbstractApplication { @Override protected void execute(OptionsHelper optionsHelper) throws Exception { String exportDest = optionsHelper.getOptionValue(options.getOption("destDir")); - boolean compress = optionsHelper.hasOption(OPTION_COMPRESS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_COMPRESS)) : true; - boolean quiet = optionsHelper.hasOption(OPTION_QUIET) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_QUIET)) : false; + boolean shouldCompress = optionsHelper.hasOption(OPTION_COMPRESS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_COMPRESS)) : true; + boolean isSubmodule = optionsHelper.hasOption(OPTION_SUBMODULE) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_SUBMODULE)) : false; if (StringUtils.isEmpty(exportDest)) { throw new RuntimeException("destDir is not set, exit directly without extracting"); @@ -88,11 +88,14 @@ public abstract class AbstractInfoExtractor extends AbstractApplication { } exportDir = new File(exportDest); - dumpBasicDiagInfo(); + if (!isSubmodule) { + dumpBasicDiagInfo(); + } + executeExtract(optionsHelper, exportDir); // compress to zip package - if (compress) { + if (shouldCompress) { File tempZipFile = File.createTempFile(packageType + "_", ".zip"); ZipFileUtils.compressZipFile(exportDir.getAbsolutePath(), tempZipFile.getAbsolutePath()); FileUtils.cleanDirectory(exportDir); @@ -103,7 +106,7 @@ public abstract class AbstractInfoExtractor extends AbstractApplication { exportDir = new File(exportDest); } - if (!quiet) { + if (!isSubmodule) { StringBuffer output = new StringBuffer(); output.append("\n========================================"); output.append("\nDump " + packageType + " package locates at: \n" + exportDir.getAbsolutePath()); @@ -118,7 +121,7 @@ public abstract class AbstractInfoExtractor extends AbstractApplication { FileUtils.writeStringToFile(new File(exportDir, "kylin_env"), output); StringBuilder basicSb = new StringBuilder(); - basicSb.append(ToolUtil.getHBaseMetaStoreId()).append("\n"); + basicSb.append("MetaStoreID: ").append(ToolUtil.getHBaseMetaStoreId()).append("\n"); basicSb.append("PackageType: ").append(packageType.toUpperCase()).append("\n"); FileUtils.writeStringToFile(new File(exportDir, "info"), basicSb.toString()); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/kylin/blob/0308fa81/tool/src/main/java/org/apache/kylin/tool/ClientEnvExtractor.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/ClientEnvExtractor.java b/tool/src/main/java/org/apache/kylin/tool/ClientEnvExtractor.java index b0e79d4..01908be 100644 --- a/tool/src/main/java/org/apache/kylin/tool/ClientEnvExtractor.java +++ b/tool/src/main/java/org/apache/kylin/tool/ClientEnvExtractor.java @@ -47,17 +47,28 @@ public class ClientEnvExtractor extends AbstractInfoExtractor { @Override protected void executeExtract(OptionsHelper optionsHelper, File exportDir) throws Exception { + // dump os info addLocalFile("/sys/kernel/mm/transparent_hugepage/defrag", "linux/transparent_hugepage"); addLocalFile("/proc/sys/vm/swappiness", "linux/swappiness"); + addLocalFile("/proc/cpuinfo", "linux/cpuinfo"); addShellOutput("lsb_release -a", "linux", "lsb_release"); - addShellOutput("ps aux|grep kylin", "kylin", "processes"); + addShellOutput("df -h", "linux", "disk_usage"); + addShellOutput("free -m", "linux", "mem_usage_mb"); + addShellOutput("top -n 1 | head -n 30", "linux", "top"); + addShellOutput("ps aux|grep kylin", "linux", "kylin_processes"); + + // dump hadoop env + addShellOutput("hadoop version", "hadoop", "version"); + addShellOutput("hbase version", "hbase", "version"); + addShellOutput("hive --version", "hive", "version"); + addShellOutput("beeline --version", "hive", "beeline_version"); } private void addLocalFile(String src, String destDir) { try { File srcFile = new File(src); File destDirFile = null; - if (StringUtils.isEmpty(destDir)) { + if (!StringUtils.isEmpty(destDir)) { destDirFile = new File(exportDir, destDir); FileUtils.forceMkdir(destDirFile); } else { @@ -73,7 +84,7 @@ public class ClientEnvExtractor extends AbstractInfoExtractor { private void addShellOutput(String cmd, String destDir, String filename) { try { File destDirFile = null; - if (StringUtils.isEmpty(destDir)) { + if (!StringUtils.isEmpty(destDir)) { destDirFile = new File(exportDir, destDir); FileUtils.forceMkdir(destDirFile); } else { http://git-wip-us.apache.org/repos/asf/kylin/blob/0308fa81/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java index bd88fe7..863171d 100644 --- a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java +++ b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java @@ -25,7 +25,6 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.io.FileUtils; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.KylinVersion; import org.apache.kylin.common.util.OptionsHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,13 +44,10 @@ public class DiagnosisInfoCLI extends AbstractInfoExtractor { @SuppressWarnings("static-access") private static final Option OPTION_INCLUDE_CLIENT = OptionBuilder.withArgName("includeClient").hasArg().isRequired(false).withDescription("Specify whether to include client info to extract. Default true.").create("includeClient"); - private KylinConfig kylinConfig; - public DiagnosisInfoCLI() { super(); - packageType = "diagnosis"; - kylinConfig = KylinConfig.getInstanceFromEnv(); + packageType = "project"; options.addOption(OPTION_PROJECT); options.addOption(OPTION_INCLUDE_CONF); @@ -72,13 +68,13 @@ public class DiagnosisInfoCLI extends AbstractInfoExtractor { boolean includeClient = optionsHelper.hasOption(OPTION_INCLUDE_CLIENT) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_CLIENT)) : true; // export cube metadata - String[] cubeMetaArgs = { "-destDir", new File(exportDir, "metadata").getAbsolutePath(), "-project", project, "-compress", "false", "-quiet", "false" }; + String[] cubeMetaArgs = { "-destDir", new File(exportDir, "metadata").getAbsolutePath(), "-project", project, "-compress", "false", "-submodule", "true" }; CubeMetaExtractor cubeMetaExtractor = new CubeMetaExtractor(); cubeMetaExtractor.execute(cubeMetaArgs); // export HBase if (includeHBase) { - String[] hbaseArgs = { "-destDir", new File(exportDir, "hbase").getAbsolutePath(), "-project", project, "-compress", "false", "-quiet", "false" }; + String[] hbaseArgs = { "-destDir", new File(exportDir, "hbase").getAbsolutePath(), "-project", project, "-compress", "false", "-submodule", "true" }; HBaseUsageExtractor hBaseUsageExtractor = new HBaseUsageExtractor(); hBaseUsageExtractor.execute(hbaseArgs); } @@ -95,7 +91,7 @@ public class DiagnosisInfoCLI extends AbstractInfoExtractor { // export client if (includeClient) { - String[] clientArgs = { "-destDir", new File(exportDir, "client").getAbsolutePath(), "-compress", "false", "-quiet", "false" }; + String[] clientArgs = { "-destDir", new File(exportDir, "client").getAbsolutePath(), "-compress", "false", "-submodule", "true" }; ClientEnvExtractor clientEnvExtractor = new ClientEnvExtractor(); clientEnvExtractor.execute(clientArgs); } @@ -107,21 +103,8 @@ public class DiagnosisInfoCLI extends AbstractInfoExtractor { logger.warn("Error in export commit id.", e); } - // export basic info - try { - File basicDir = new File(exportDir, "basic"); - FileUtils.forceMkdir(basicDir); - String output = KylinVersion.getKylinClientInformation(); - FileUtils.writeStringToFile(new File(basicDir, "client"), output + "\n"); - output = ToolUtil.getHBaseMetaStoreId(); - FileUtils.writeStringToFile(new File(basicDir, "client"), output, true); - - } catch (Exception e) { - logger.warn("Error in export process info.", e); - } - // export logs - String[] logsArgs = { "-destDir", new File(exportDir, "logs").getAbsolutePath(), "-compress", "false", "-quiet", "false" }; + String[] logsArgs = { "-destDir", new File(exportDir, "logs").getAbsolutePath(), "-compress", "false", "-submodule", "true" }; KylinLogExtractor logExtractor = new KylinLogExtractor(); logExtractor.execute(logsArgs); } http://git-wip-us.apache.org/repos/asf/kylin/blob/0308fa81/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java b/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java index 85fd539..45e3072 100644 --- a/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java +++ b/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java @@ -101,7 +101,7 @@ public class JobDiagnosisInfoCLI extends AbstractInfoExtractor { if (!StringUtils.isEmpty(cubeName)) { File metaDir = new File(exportDir, "cube"); FileUtils.forceMkdir(metaDir); - String[] cubeMetaArgs = { "-cube", cubeName, "-destDir", new File(metaDir, cubeName).getAbsolutePath(), "-includeJobs", "false", "-compress", "false", "-quiet", "false" }; + String[] cubeMetaArgs = { "-cube", cubeName, "-destDir", new File(metaDir, cubeName).getAbsolutePath(), "-includeJobs", "false", "-compress", "false", "-submodule", "true" }; logger.info("Start to extract related cube: " + StringUtils.join(cubeMetaArgs)); CubeMetaExtractor cubeMetaExtractor = new CubeMetaExtractor(); @@ -120,13 +120,13 @@ public class JobDiagnosisInfoCLI extends AbstractInfoExtractor { } if (includeClient) { - String[] clientArgs = { "-destDir", new File(exportDir, "client").getAbsolutePath(), "-compress", "false", "-quiet", "false" }; + String[] clientArgs = { "-destDir", new File(exportDir, "client").getAbsolutePath(), "-compress", "false", "-submodule", "true" }; ClientEnvExtractor clientEnvExtractor = new ClientEnvExtractor(); clientEnvExtractor.execute(clientArgs); } // export kylin logs - String[] logsArgs = { "-destDir", new File(exportDir, "logs").getAbsolutePath(), "-compress", "false", "-quiet", "false" }; + String[] logsArgs = { "-destDir", new File(exportDir, "logs").getAbsolutePath(), "-compress", "false", "-submodule", "true" }; KylinLogExtractor logExtractor = new KylinLogExtractor(); logExtractor.execute(logsArgs); } http://git-wip-us.apache.org/repos/asf/kylin/blob/0308fa81/tool/src/main/java/org/apache/kylin/tool/ToolUtil.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/ToolUtil.java b/tool/src/main/java/org/apache/kylin/tool/ToolUtil.java index 24a7a37..aa8810c 100644 --- a/tool/src/main/java/org/apache/kylin/tool/ToolUtil.java +++ b/tool/src/main/java/org/apache/kylin/tool/ToolUtil.java @@ -49,7 +49,7 @@ public class ToolUtil { try (final HBaseAdmin hbaseAdmin = new HBaseAdmin(HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration()))) { final String metaStoreName = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix(); final HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(metaStoreName)); - return "MetaStore UUID: " + desc.getValue(HBaseConnection.HTABLE_UUID_TAG); + return desc.getValue(HBaseConnection.HTABLE_UUID_TAG); } }
