This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new 168fa86 Add built instance in Job info
168fa86 is described below
commit 168fa86ebe8bbd09642f894da0312ebc623f1ada
Author: Liu Shaohui <[email protected]>
AuthorDate: Fri Jan 11 13:43:32 2019 +0800
Add built instance in Job info
Summary: Ref T13014
Reviewers: zhangxu16, wubin1, hubei
Reviewed By: zhangxu16, wubin1
Subscribers: cloud-computing
Maniphest Tasks: T13014
Differential Revision: https://phabricator.d.xiaomi.net/D135075
---
.../org/apache/kylin/common/lock/DistributedLockFactory.java | 2 +-
core-job/src/main/java/org/apache/kylin/job/JobInstance.java | 11 +++++++++++
.../org/apache/kylin/job/execution/AbstractExecutable.java | 9 +++++++++
.../apache/kylin/job/execution/DefaultChainedExecutable.java | 6 ++++--
.../org/apache/kylin/job/execution/ExecutableManager.java | 2 +-
.../org/apache/kylin/engine/mr/common/JobInfoConverter.java | 2 ++
.../main/java/org/apache/kylin/rest/service/JobService.java | 7 ++++++-
.../main/java/org/apache/kylin/tool/JobInstanceExtractor.java | 1 +
webapp/app/partials/jobs/job_steps.html | 4 ++++
9 files changed, 39 insertions(+), 5 deletions(-)
diff --git
a/core-common/src/main/java/org/apache/kylin/common/lock/DistributedLockFactory.java
b/core-common/src/main/java/org/apache/kylin/common/lock/DistributedLockFactory.java
index e03d0e8..05597e8 100644
---
a/core-common/src/main/java/org/apache/kylin/common/lock/DistributedLockFactory.java
+++
b/core-common/src/main/java/org/apache/kylin/common/lock/DistributedLockFactory.java
@@ -37,7 +37,7 @@ public abstract class DistributedLockFactory {
return Thread.currentThread().getId() + "-" + processAndHost();
}
- private static String processAndHost() {
+ public static String processAndHost() {
byte[] bytes =
ManagementFactory.getRuntimeMXBean().getName().getBytes(StandardCharsets.UTF_8);
return new String(bytes, StandardCharsets.UTF_8);
}
diff --git a/core-job/src/main/java/org/apache/kylin/job/JobInstance.java
b/core-job/src/main/java/org/apache/kylin/job/JobInstance.java
index a2c3459..ef649ab 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JobInstance.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JobInstance.java
@@ -72,6 +72,8 @@ public class JobInstance extends RootPersistentEntity
implements Comparable<JobI
private String submitter;
@JsonProperty("job_status")
private JobStatusEnum status;
+ @JsonProperty("build_instance")
+ private String buildInstance;
public JobStep getRunningStep() {
for (JobStep step : this.getSteps()) {
@@ -289,6 +291,14 @@ public class JobInstance extends RootPersistentEntity
implements Comparable<JobI
this.submitter = submitter;
}
+ public String getBuildInstance() {
+ return buildInstance;
+ }
+
+ public void setBuildInstance(String buildInstance) {
+ this.buildInstance = buildInstance;
+ }
+
@JsonIgnoreProperties(ignoreUnknown = true)
public static class JobStep implements Comparable<JobStep> {
@@ -451,6 +461,7 @@ public class JobInstance extends RootPersistentEntity
implements Comparable<JobI
this.runAsync = runAsync;
}
+
/**
* @return the jobInstance
*/
diff --git
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index 8548781..e808084 100644
---
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -55,6 +55,7 @@ public abstract class AbstractExecutable implements
Executable, Idempotent {
protected static final String START_TIME = "startTime";
protected static final String END_TIME = "endTime";
protected static final String INTERRUPT_TIME = "interruptTime";
+ protected static final String BUILD_INSTANCE = "buildInstance";
protected static final Logger logger =
LoggerFactory.getLogger(AbstractExecutable.class);
public static final String NO_NEED_TO_SEND_EMAIL_USER_LIST_IS_EMPTY = "no
need to send email, user list is empty";
@@ -375,6 +376,14 @@ public abstract class AbstractExecutable implements
Executable, Idempotent {
return getExtraInfoAsLong(getOutput(), key, defaultValue);
}
+ public static String getBuildInstance(Output output) {
+ final String str = output.getExtra().get(BUILD_INSTANCE);
+ if (str != null) {
+ return str;
+ }
+ return "unknown";
+ }
+
public static long getStartTime(Output output) {
return getExtraInfoAsLong(output, START_TIME, 0L);
}
diff --git
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
index b912ecc..3bcd8ca 100644
---
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
+++
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
@@ -25,6 +25,7 @@ import java.util.Locale;
import java.util.Map;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.lock.DistributedLockFactory;
import org.apache.kylin.job.exception.ExecuteException;
import com.google.common.collect.Lists;
@@ -76,10 +77,11 @@ public class DefaultChainedExecutable extends
AbstractExecutable implements Chai
@Override
protected void onExecuteStart(ExecutableContext executableContext) {
final long startTime = getStartTime();
+ Map<String, String> info = Maps.newHashMap();
+ info.put(BUILD_INSTANCE, DistributedLockFactory.processAndHost());
if (startTime > 0) {
- getManager().updateJobOutput(getId(), ExecutableState.RUNNING,
null, null);
+ getManager().updateJobOutput(getId(), ExecutableState.RUNNING,
info, null);
} else {
- Map<String, String> info = Maps.newHashMap();
info.put(START_TIME, Long.toString(System.currentTimeMillis()));
getManager().updateJobOutput(getId(), ExecutableState.RUNNING,
info, null);
}
diff --git
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index b1244b5..22aa859 100644
---
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -444,7 +444,7 @@ public class ExecutableManager {
jobOutput.setStatus(newStatus.toString());
}
if (info != null) {
- jobOutput.setInfo(info);
+ jobOutput.getInfo().putAll(info);
}
if (output != null) {
jobOutput.setContent(output);
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
index 19fdca8..1b9b2d6 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
@@ -87,6 +87,7 @@ public class JobInfoConverter {
result.setUuid(job.getId());
result.setType(CubeBuildTypeEnum.BUILD);
result.setStatus(parseToJobStatus(output.getState()));
+ result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
result.setMrWaiting(AbstractExecutable.getExtraInfoAsLong(output,
CubingJob.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
result.setExecStartTime(AbstractExecutable.getStartTime(output));
result.setExecEndTime(AbstractExecutable.getEndTime(output));
@@ -121,6 +122,7 @@ public class JobInfoConverter {
result.setUuid(job.getId());
result.setType(CubeBuildTypeEnum.CHECKPOINT);
result.setStatus(parseToJobStatus(output.getState()));
+ result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
result.setExecStartTime(AbstractExecutable.getStartTime(output));
result.setExecEndTime(AbstractExecutable.getEndTime(output));
result.setExecInterruptTime(AbstractExecutable.getInterruptTime(output));
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 28cca02..46feea0 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -470,6 +470,7 @@ public class JobService extends BasicService implements
InitializingBean {
CubingJob cubeJob = (CubingJob) job;
CubeInstance cube =
CubeManager.getInstance(KylinConfig.getInstanceFromEnv())
.getCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));
+ Output output = cubeJob.getOutput();
final JobInstance result = new JobInstance();
result.setName(job.getName());
if (cube != null) {
@@ -487,6 +488,7 @@ public class JobService extends BasicService implements
InitializingBean {
result.setType(CubeBuildTypeEnum.BUILD);
result.setStatus(JobInfoConverter.parseToJobStatus(job.getStatus()));
result.setMrWaiting(cubeJob.getMapReduceWaitTime() / 1000);
+ result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
result.setDuration(cubeJob.getDuration() / 1000);
for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
AbstractExecutable task = cubeJob.getTasks().get(i);
@@ -499,7 +501,7 @@ public class JobService extends BasicService implements
InitializingBean {
if (job == null) {
return null;
}
-
+ Output output = job.getOutput();
final JobInstance result = new JobInstance();
result.setName(job.getName());
result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams()));
@@ -509,6 +511,7 @@ public class JobService extends BasicService implements
InitializingBean {
result.setUuid(job.getId());
result.setType(CubeBuildTypeEnum.BUILD);
result.setStatus(JobInfoConverter.parseToJobStatus(job.getStatus()));
+ result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
result.setDuration(job.getDuration() / 1000);
for (int i = 0; i < job.getTasks().size(); ++i) {
AbstractExecutable task = job.getTasks().get(i);
@@ -528,6 +531,7 @@ public class JobService extends BasicService implements
InitializingBean {
}
CheckpointExecutable checkpointExecutable = (CheckpointExecutable) job;
+ Output output = checkpointExecutable.getOutput();
final JobInstance result = new JobInstance();
result.setName(job.getName());
result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams()));
@@ -537,6 +541,7 @@ public class JobService extends BasicService implements
InitializingBean {
result.setUuid(job.getId());
result.setType(CubeBuildTypeEnum.CHECKPOINT);
result.setStatus(JobInfoConverter.parseToJobStatus(job.getStatus()));
+ result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
result.setDuration(job.getDuration() / 1000);
for (int i = 0; i < checkpointExecutable.getTasks().size(); ++i) {
AbstractExecutable task = checkpointExecutable.getTasks().get(i);
diff --git a/tool/src/main/java/org/apache/kylin/tool/JobInstanceExtractor.java
b/tool/src/main/java/org/apache/kylin/tool/JobInstanceExtractor.java
index 18ace6f..5b58c4a 100644
--- a/tool/src/main/java/org/apache/kylin/tool/JobInstanceExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/JobInstanceExtractor.java
@@ -145,6 +145,7 @@ public class JobInstanceExtractor extends
AbstractInfoExtractor {
result.setType(CubeBuildTypeEnum.BUILD);
result.setStatus(parseToJobStatus(output.getState()));
result.setMrWaiting(AbstractExecutable.getExtraInfoAsLong(output,
CubingJob.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
+ result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
result.setExecStartTime(AbstractExecutable.getStartTime(output));
result.setExecEndTime(AbstractExecutable.getEndTime(output));
result.setExecInterruptTime(AbstractExecutable.getInterruptTime(output));
diff --git a/webapp/app/partials/jobs/job_steps.html
b/webapp/app/partials/jobs/job_steps.html
index f451805..0c64e15 100644
--- a/webapp/app/partials/jobs/job_steps.html
+++ b/webapp/app/partials/jobs/job_steps.html
@@ -54,6 +54,10 @@
</td>
</tr>
<tr>
+ <td><b>Built By</b></td>
+ <td>{{state.selectedJob.build_instance}}</td>
+ </tr>
+ <tr>
<td><b>Duration</b></td>
<td>{{(state.selectedJob.duration/60 | number:2) +
' mins'}}</td>
</tr>
