Author: acmurthy
Date: Sat Apr 6 05:30:56 2013
New Revision: 1465188
URL: http://svn.apache.org/r1465188
Log:
MAPREDUCE-5129. Allow tags to JobHistory for deeper analytics. Contributed by
Billie Rinaldi.
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1465188&r1=1465187&r2=1465188&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Sat Apr 6
05:30:56 2013
@@ -166,6 +166,9 @@ Release 2.0.5-alpha - UNRELEASED
IMPROVEMENTS
+ MAPREDUCE-5129. Allow tags to JobHistory for deeper analytics. (billie via
+ acmurthy)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1465188&r1=1465187&r2=1465188&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
Sat Apr 6 05:30:56 2013
@@ -1305,7 +1305,8 @@ public class JobImpl implements org.apac
job.conf.get(MRJobConfig.WORKFLOW_ID, ""),
job.conf.get(MRJobConfig.WORKFLOW_NAME, ""),
job.conf.get(MRJobConfig.WORKFLOW_NODE_NAME, ""),
- getWorkflowAdjacencies(job.conf));
+ getWorkflowAdjacencies(job.conf),
+ job.conf.get(MRJobConfig.WORKFLOW_TAGS, ""));
job.eventHandler.handle(new JobHistoryEvent(job.jobId, jse));
//TODO JH Verify jobACLs, UserName via UGI?
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java?rev=1465188&r1=1465187&r2=1465188&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
Sat Apr 6 05:30:56 2013
@@ -114,6 +114,7 @@ public class TestJobImpl {
conf.set(MRJobConfig.WORKFLOW_NODE_NAME, "testNodeName");
conf.set(MRJobConfig.WORKFLOW_ADJACENCY_PREFIX_STRING + "key1", "value1");
conf.set(MRJobConfig.WORKFLOW_ADJACENCY_PREFIX_STRING + "key2", "value2");
+ conf.set(MRJobConfig.WORKFLOW_TAGS, "tag1,tag2");
AsyncDispatcher dispatcher = new AsyncDispatcher();
@@ -126,7 +127,8 @@ public class TestJobImpl {
commitHandler.start();
JobSubmittedEventHandler jseHandler = new
JobSubmittedEventHandler("testId",
- "testName", "testNodeName", "\"key2\"=\"value2\" \"key1\"=\"value1\"
");
+ "testName", "testNodeName", "\"key2\"=\"value2\" \"key1\"=\"value1\" ",
+ "tag1,tag2");
dispatcher.register(EventType.class, jseHandler);
JobImpl job = createStubbedJob(conf, dispatcher, 0);
job.handle(new JobEvent(job.getID(), JobEventType.JOB_INIT));
@@ -706,14 +708,18 @@ public class TestJobImpl {
private String workflowAdjacencies;
+ private String workflowTags;
+
private Boolean assertBoolean;
public JobSubmittedEventHandler(String workflowId, String workflowName,
- String workflowNodeName, String workflowAdjacencies) {
+ String workflowNodeName, String workflowAdjacencies,
+ String workflowTags) {
this.workflowId = workflowId;
this.workflowName = workflowName;
this.workflowNodeName = workflowNodeName;
this.workflowAdjacencies = workflowAdjacencies;
+ this.workflowTags = workflowTags;
assertBoolean = null;
}
@@ -739,6 +745,10 @@ public class TestJobImpl {
setAssertValue(false);
return;
}
+ if (!workflowTags.equals(jsEvent.getWorkflowTags())) {
+ setAssertValue(false);
+ return;
+ }
setAssertValue(true);
}
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr?rev=1465188&r1=1465187&r2=1465188&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr
Sat Apr 6 05:30:56 2013
@@ -95,7 +95,8 @@
{"name": "workflowId", "type": "string"},
{"name": "workflowName", "type": "string"},
{"name": "workflowNodeName", "type": "string"},
- {"name": "workflowAdjacencies", "type": "string"}
+ {"name": "workflowAdjacencies", "type": "string"},
+ {"name": "workflowTags", "type": "string"}
]
},
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1465188&r1=1465187&r2=1465188&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
Sat Apr 6 05:30:56 2013
@@ -664,6 +664,8 @@ public interface MRJobConfig {
public static final String WORKFLOW_ADJACENCY_PREFIX_PATTERN =
"^mapreduce\\.workflow\\.adjacency\\..+";
+ public static final String WORKFLOW_TAGS = "mapreduce.workflow.tags";
+
/**
* The maximum number of application attempts.
* It is a application-specific setting.
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java?rev=1465188&r1=1465187&r2=1465188&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
Sat Apr 6 05:30:56 2013
@@ -75,6 +75,31 @@ public class JobSubmittedEvent implement
Map<JobACL, AccessControlList> jobACLs, String jobQueueName,
String workflowId, String workflowName, String workflowNodeName,
String workflowAdjacencies) {
+ this(id, jobName, userName, submitTime, jobConfPath, jobACLs,
+ jobQueueName, workflowId, workflowName, workflowNodeName,
+ workflowAdjacencies, "");
+ }
+
+ /**
+ * Create an event to record job submission
+ * @param id The job Id of the job
+ * @param jobName Name of the job
+ * @param userName Name of the user who submitted the job
+ * @param submitTime Time of submission
+ * @param jobConfPath Path of the Job Configuration file
+ * @param jobACLs The configured acls for the job.
+ * @param jobQueueName The job-queue to which this job was submitted to
+ * @param workflowId The Id of the workflow
+ * @param workflowName The name of the workflow
+ * @param workflowNodeName The node name of the workflow
+ * @param workflowAdjacencies The adjacencies of the workflow
+ * @param workflowTags Comma-separated tags for the workflow
+ */
+ public JobSubmittedEvent(JobID id, String jobName, String userName,
+ long submitTime, String jobConfPath,
+ Map<JobACL, AccessControlList> jobACLs, String jobQueueName,
+ String workflowId, String workflowName, String workflowNodeName,
+ String workflowAdjacencies, String workflowTags) {
datum.jobid = new Utf8(id.toString());
datum.jobName = new Utf8(jobName);
datum.userName = new Utf8(userName);
@@ -101,6 +126,9 @@ public class JobSubmittedEvent implement
if (workflowAdjacencies != null) {
datum.workflowAdjacencies = new Utf8(workflowAdjacencies);
}
+ if (workflowTags != null) {
+ datum.workflowTags = new Utf8(workflowTags);
+ }
}
JobSubmittedEvent() {}
@@ -168,6 +196,13 @@ public class JobSubmittedEvent implement
}
return null;
}
+ /** Get the workflow tags */
+ public String getWorkflowTags() {
+ if (datum.workflowTags != null) {
+ return datum.workflowTags.toString();
+ }
+ return null;
+ }
/** Get the event type */
public EventType getEventType() { return EventType.JOB_SUBMITTED; }
