KYLIN-1614 Scripts to package diagnosis for 1.3 and 1.5
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/277e6d62 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/277e6d62 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/277e6d62 Branch: refs/heads/1.5.x-HBase1.x Commit: 277e6d6285078a369c6a7fc388e899716a8eac65 Parents: eac9803 Author: lidongsjtu <lid...@apache.org> Authored: Sun May 8 13:56:17 2016 +0800 Committer: lidongsjtu <lid...@apache.org> Committed: Sun May 8 18:14:21 2016 +0800 ---------------------------------------------------------------------- build/bin/kylin.sh | 2 +- build/deploy/diag.sh | 16 +++++++++++----- build/script/package_diag.sh | 3 ++- tool/pom.xml | 15 +++------------ .../org/apache/kylin/tool/CubeMetaExtractor.java | 6 +++++- .../org/apache/kylin/tool/JobDiagnosisInfoCLI.java | 14 +++++--------- 6 files changed, 27 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/277e6d62/build/bin/kylin.sh ---------------------------------------------------------------------- diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh index 7ceae69..0e503ee 100644 --- a/build/bin/kylin.sh +++ b/build/bin/kylin.sh @@ -217,7 +217,7 @@ then export HBASE_CLASSPATH=${KYLIN_HOME}/lib/* - if [ ${#patient} -gt 31 ]; then + if [ ${#patient} -eq 36 ]; then exec hbase ${KYLIN_EXTRA_START_OPTS} -Dlog4j.configuration=kylin-log4j.properties org.apache.kylin.tool.JobDiagnosisInfoCLI -jobId $patient -destDir $destDir else exec hbase ${KYLIN_EXTRA_START_OPTS} -Dlog4j.configuration=kylin-log4j.properties org.apache.kylin.tool.DiagnosisInfoCLI -project $patient -destDir $destDir http://git-wip-us.apache.org/repos/asf/kylin/blob/277e6d62/build/deploy/diag.sh ---------------------------------------------------------------------- diff --git a/build/deploy/diag.sh b/build/deploy/diag.sh index 1deeded..3c37c4b 100644 --- a/build/deploy/diag.sh +++ b/build/deploy/diag.sh @@ -24,7 +24,7 @@ dir=$(dirname ${0}) export KYLIN_HOME=${dir}/../ source ${dir}/check-env.sh -if [ $# -gt 1 ] +if [ $# -eq 1 ] || [ $# -eq 2 ] then patient="$1" if [ -z "$patient" ] @@ -49,13 +49,19 @@ then fi export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH} - if [ -f "${KYLIN_HOME}/lib/kylin-diagnosis-*.jar" ]; then - export HBASE_CLASSPATH=$HBASE_CLASSPATH:${KYLIN_HOME}/lib/kylin-diagnosis-*.jar + diagJar=`ls ${KYLIN_HOME}/lib/kylin-diagnosis-*.jar` + if [ -f "${diagJar}" ]; then + if [ -f "${KYLIN_HOME}/commit_SHA1" ]; then + export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${diagJar}:${KYLIN_HOME}/lib/* + else + export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${KYLIN_HOME}/lib/* + fi else - export HBASE_CLASSPATH=$HBASE_CLASSPATH:${KYLIN_HOME}/lib/* + echo "missing diagnosis jar file." + exit 1 fi - if [ ${#patient} -gt 31 ]; then + if [ ${#patient} -eq 36 ]; then exec hbase ${KYLIN_EXTRA_START_OPTS} -Dlog4j.configuration=kylin-log4j.properties org.apache.kylin.tool.JobDiagnosisInfoCLI -jobId $patient -destDir $destDir else exec hbase ${KYLIN_EXTRA_START_OPTS} -Dlog4j.configuration=kylin-log4j.properties org.apache.kylin.tool.DiagnosisInfoCLI -project $patient -destDir $destDir http://git-wip-us.apache.org/repos/asf/kylin/blob/277e6d62/build/script/package_diag.sh ---------------------------------------------------------------------- diff --git a/build/script/package_diag.sh b/build/script/package_diag.sh index 413bca8..5de1590 100755 --- a/build/script/package_diag.sh +++ b/build/script/package_diag.sh @@ -84,8 +84,9 @@ package_name=apache-kylin-${version}-diag cd build/ rm -rf ${package_name} mkdir ${package_name} +mkdir ${package_name}/bin cp -r lib ${package_name} -cp deploy/diag.sh ${package_name} +cp deploy/diag.sh ${package_name}/bin/ rm -rf lib find ${package_name} -type d -exec chmod 755 {} \; find ${package_name} -type f -exec chmod 644 {} \; http://git-wip-us.apache.org/repos/asf/kylin/blob/277e6d62/tool/pom.xml ---------------------------------------------------------------------- diff --git a/tool/pom.xml b/tool/pom.xml index 3f077de..3041ae2 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -31,11 +31,6 @@ <dependencies> <dependency> <groupId>org.apache.kylin</groupId> - <artifactId>kylin-source-kafka</artifactId> - <version>${project.parent.version}</version> - </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> <artifactId>kylin-storage-hbase</artifactId> <version>${project.parent.version}</version> </dependency> @@ -64,7 +59,6 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-shade-plugin</artifactId> <version>2.3</version> - <executions> <execution> <phase>package</phase> @@ -72,15 +66,12 @@ <goal>shade</goal> </goals> <configuration> - <minimizeJar>false</minimizeJar> <shadedArtifactAttached>true</shadedArtifactAttached> <shadedClassifierName>assembly</shadedClassifierName> <artifactSet> - <excludes> - <exclude>io.netty:*</exclude> - <exclude>org.apache.zookeeper:*</exclude> - <exclude>net.sf.ehcache:*</exclude> - </excludes> + <includes> + <include>org.apache.kylin:*</include> + </includes> </artifactSet> <filters> <filter> http://git-wip-us.apache.org/repos/asf/kylin/blob/277e6d62/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java index a8b732b..3de346d 100644 --- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java +++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java @@ -234,6 +234,10 @@ public class CubeMetaExtractor extends AbstractInfoExtractor { return ResourceStore.CUBE_DESC_RESOURCE_ROOT + "/" + descName + MetadataConstants.FILE_SURFIX; } + private static String concatCubeSegmentStatisticsResourcePath(String cubeName, String cubeSegmentId) { + return ResourceStore.CUBE_STATISTICS_ROOT + "/" + cubeName + "/" + cubeSegmentId + ".seq"; + } + private void retrieveResourcePath(IRealization realization) { logger.info("Deal with realization {} of type {}", realization.getName(), realization.getType()); @@ -260,7 +264,7 @@ public class CubeMetaExtractor extends AbstractInfoExtractor { if (includeSegments) { addRequired(CubeInstance.concatResourcePath(cube.getName())); for (CubeSegment segment : cube.getSegments(SegmentStatusEnum.READY)) { - addRequired(segment.getStatisticsResourcePath()); + addRequired(concatCubeSegmentStatisticsResourcePath(cube.getName(), segment.getUuid())); if (includeSegmentDetails) { for (String dictPat : segment.getDictionaryPaths()) { addRequired(dictPat); http://git-wip-us.apache.org/repos/asf/kylin/blob/277e6d62/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java b/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java index 45e3072..a479f9e 100644 --- a/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java +++ b/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java @@ -28,12 +28,11 @@ import org.apache.commons.cli.OptionBuilder; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.common.persistence.ResourceTool; import org.apache.kylin.common.util.OptionsHelper; import org.apache.kylin.job.constant.ExecutableConstants; import org.apache.kylin.job.dao.ExecutableDao; import org.apache.kylin.job.dao.ExecutablePO; +import org.apache.kylin.tool.util.ResourceStoreUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -138,14 +137,11 @@ public class JobDiagnosisInfoCLI extends AbstractInfoExtractor { } try { - ResourceStore src = ResourceStore.getStore(KylinConfig.getInstanceFromEnv()); - ResourceStore dst = ResourceStore.getStore(KylinConfig.createInstanceFromUri(destDir.getAbsolutePath())); + KylinConfig srcConfig = KylinConfig.getInstanceFromEnv(); + KylinConfig dstConfig = KylinConfig.createInstanceFromUri(destDir.getAbsolutePath()); - for (String path : requiredResources) { - ResourceTool.copyR(src, dst, path); - } - - } catch (IOException e) { + ResourceStoreUtil.copy(srcConfig, dstConfig, requiredResources); + } catch (Exception e) { throw new RuntimeException("Failed to extract job resources. ", e); } }
