Repository: kylin Updated Branches: refs/heads/1.4-rc 08b2051f8 -> 85113c11f
Add Env and Project Info in job email notification Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/85113c11 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/85113c11 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/85113c11 Branch: refs/heads/1.4-rc Commit: 85113c11f443c8ee702c5a8d201b926349e80306 Parents: 08b2051 Author: Zhong <yangzh...@lm-shc-16501214.corp.ebay.com> Authored: Mon Apr 18 17:39:57 2016 +0800 Committer: Zhong <yangzh...@lm-shc-16501214.corp.ebay.com> Committed: Mon Apr 18 17:39:57 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 11 ++-- .../kylin/job/constant/ExecutableConstants.java | 2 +- .../org/apache/kylin/engine/mr/CubingJob.java | 56 ++++++++++++++++---- 3 files changed, 52 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/85113c11/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 9bbcbb4..6f1fccc 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -173,10 +173,7 @@ public class KylinConfigBase implements Serializable { if (!root.endsWith("/")) { root += "/"; } - return new StringBuffer(root) - .append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')) - .append("/") - .toString(); + return new StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')).append("/").toString(); } public CliCommandExecutor getCliCommandExecutor() throws IOException { @@ -485,7 +482,7 @@ public class KylinConfigBase implements Serializable { public boolean isGetJobStatusWithKerberos() { return Boolean.valueOf(this.getOptional("kylin.job.status.with.kerberos", "false")); } - + public boolean isTransformPathToMasterNN() { return Boolean.valueOf(this.getOptional("kylin.transform.hdfs.path.enable", "false")); } @@ -526,6 +523,10 @@ public class KylinConfigBase implements Serializable { return getOptional("mail.sender", ""); } + public String getDeployEnv() { + return getOptional("deploy.env", "DEV"); + } + public String toString() { return getMetadataUrl(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/85113c11/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java index d370b0d..ab2fed7 100644 --- a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java +++ b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java @@ -75,5 +75,5 @@ public final class ExecutableConstants { public static final String PROP_JOB_KILLED = "jobKilled"; public static final String PROP_JOB_RUNTIME_FLOWS = "jobFlows"; - public static final String NOTIFY_EMAIL_TEMPLATE = "<div><b>Build Result of Job ${job_name}</b><pre><ul>" + "<li>Build Result: <b>${result}</b></li>" + "<li>Job Engine: ${job_engine}</li>" + "<li>Cube Name: ${cube_name}</li>" + "<li>Source Records Count: ${source_records_count}</li>" + "<li>Start Time: ${start_time}</li>" + "<li>Duration: ${duration}</li>" + "<li>MR Waiting: ${mr_waiting}</li>" + "<li>Last Update Time: ${last_update_time}</li>" + "<li>Submitter: ${submitter}</li>" + "<li>Error Log: ${error_log}</li>" + "</ul></pre><div/>"; + public static final String NOTIFY_EMAIL_TEMPLATE = "<div><b>Build Result of Job ${job_name}</b><pre><ul>" + "<li>Build Result: <b>${result}</b></li>" + "<li>Job Engine: ${job_engine}</li>" + "<li>Env: ${env_name}</li>" + "<li>Project: ${project_name}</li>" + "<li>Cube Name: ${cube_name}</li>" + "<li>Source Records Count: ${source_records_count}</li>" + "<li>Start Time: ${start_time}</li>" + "<li>Duration: ${duration}</li>" + "<li>MR Waiting: ${mr_waiting}</li>" + "<li>Last Update Time: ${last_update_time}</li>" + "<li>Submitter: ${submitter}</li>" + "<li>Error Log: ${error_log}</li>" + "</ul></pre><div/>"; } http://git-wip-us.apache.org/repos/asf/kylin/blob/85113c11/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java index 979ff75..1353fe8 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java @@ -30,6 +30,7 @@ import java.util.regex.Matcher; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.StringUtil; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; @@ -43,11 +44,13 @@ import org.apache.kylin.job.execution.ExecutableContext; import org.apache.kylin.job.execution.ExecutableState; import org.apache.kylin.job.execution.ExecuteResult; import org.apache.kylin.job.execution.Output; +import org.apache.kylin.metadata.project.ProjectInstance; +import org.apache.kylin.metadata.project.ProjectManager; /** */ public class CubingJob extends DefaultChainedExecutable { - + public static enum AlgorithmEnum { LAYER, INMEM } @@ -58,6 +61,8 @@ public class CubingJob extends DefaultChainedExecutable { public static final String CUBE_SIZE_BYTES = "byteSizeBytes"; public static final String MAP_REDUCE_WAIT_TIME = "mapReduceWaitTime"; + private static final String DEPLOY_ENV_NAME = "envName"; + private static final String PROJECT_INSTANCE_NAME = "projectName"; private static final String CUBE_INSTANCE_NAME = "cubeName"; private static final String SEGMENT_ID = "segmentId"; @@ -70,10 +75,21 @@ public class CubingJob extends DefaultChainedExecutable { } private static CubingJob initCubingJob(CubeSegment seg, String jobType, String submitter, JobEngineConfig config) { + KylinConfig kylinConfig = config.getConfig(); + CubeInstance cube = seg.getCubeInstance(); + List<ProjectInstance> projList = ProjectManager.getInstance(kylinConfig).findProjects(cube.getType(),cube.getName()); + if(projList==null || projList.size()==0){ + throw new RuntimeException("Cannot find the project containing the cube " +cube.getName()+"!!!"); + }else if(projList.size()>=2){ + throw new RuntimeException("Find more than one project containing the cube "+cube.getName()+". It does't meet the uniqueness requirement!!! "); + } + CubingJob result = new CubingJob(); SimpleDateFormat format = new SimpleDateFormat("z yyyy-MM-dd HH:mm:ss"); format.setTimeZone(TimeZone.getTimeZone(config.getTimeZone())); - result.setCubeName(seg.getCubeInstance().getName()); + result.setDeployEnvName(kylinConfig.getDeployEnv()); + result.setProjectName(projList.get(0).getName()); + result.setCubeName(cube.getName()); result.setSegmentId(seg.getUuid()); result.setName(seg.getCubeInstance().getName() + " - " + seg.getName() + " - " + jobType + " - " + format.format(new Date(System.currentTimeMillis()))); result.setSubmitter(submitter); @@ -85,6 +101,22 @@ public class CubingJob extends DefaultChainedExecutable { super(); } + void setDeployEnvName(String name) { + setParam(DEPLOY_ENV_NAME, name); + } + + public String getDeployEnvName() { + return getParam(DEPLOY_ENV_NAME); + } + + void setProjectName(String name) { + setParam(PROJECT_INSTANCE_NAME, name); + } + + public String getProjectName() { + return getParam(PROJECT_INSTANCE_NAME); + } + void setCubeName(String name) { setParam(CUBE_INSTANCE_NAME, name); } @@ -131,6 +163,8 @@ public class CubingJob extends DefaultChainedExecutable { String content = ExecutableConstants.NOTIFY_EMAIL_TEMPLATE; content = content.replaceAll("\\$\\{job_name\\}", getName()); content = content.replaceAll("\\$\\{result\\}", state.toString()); + content = content.replaceAll("\\$\\{env_name\\}", getDeployEnvName()); + content = content.replaceAll("\\$\\{project_name\\}", getProjectName()); content = content.replaceAll("\\$\\{cube_name\\}", getCubeName()); content = content.replaceAll("\\$\\{source_records_count\\}", String.valueOf(findSourceRecordCount())); content = content.replaceAll("\\$\\{start_time\\}", new Date(getStartTime()).toString()); @@ -147,7 +181,7 @@ public class CubingJob extends DefaultChainedExecutable { logger.warn(e.getLocalizedMessage(), e); } - String title = "[" + state.toString() + "] - [Kylin Cube Build Job]-" + getCubeName(); + String title = "[" + state.toString() + "] - [" + getDeployEnvName() + "] - [" + getProjectName() + "] - " + getCubeName(); return Pair.of(title, content); } @@ -174,11 +208,11 @@ public class CubingJob extends DefaultChainedExecutable { public void setMapReduceWaitTime(long t) { addExtraInfo(MAP_REDUCE_WAIT_TIME, t + ""); } - + public void setAlgorithm(AlgorithmEnum alg) { addExtraInfo("algorithm", alg.name()); } - + public AlgorithmEnum getAlgorithm() { String alg = getExtraInfo().get("algorithm"); return alg == null ? null : AlgorithmEnum.valueOf(alg); @@ -187,11 +221,11 @@ public class CubingJob extends DefaultChainedExecutable { public boolean isLayerCubing() { return AlgorithmEnum.LAYER == getAlgorithm(); } - + public boolean isInMemCubing() { return AlgorithmEnum.INMEM == getAlgorithm(); } - + public long findSourceRecordCount() { return Long.parseLong(findExtraInfo(SOURCE_RECORD_COUNT, "0")); } @@ -204,7 +238,7 @@ public class CubingJob extends DefaultChainedExecutable { // look for the info BACKWARD, let the last step that claims the cube size win return Long.parseLong(findExtraInfoBackward(CUBE_SIZE_BYTES, "0")); } - + public String findExtraInfo(String key, String dft) { return findExtraInfo(key, dft, false); } @@ -212,14 +246,14 @@ public class CubingJob extends DefaultChainedExecutable { public String findExtraInfoBackward(String key, String dft) { return findExtraInfo(key, dft, true); } - + private String findExtraInfo(String key, String dft, boolean backward) { ArrayList<AbstractExecutable> tasks = new ArrayList<AbstractExecutable>(getTasks()); - + if (backward) { Collections.reverse(tasks); } - + for (AbstractExecutable child : tasks) { Output output = executableManager.getOutput(child.getId()); String value = output.getExtra().get(key);