Author: olga
Date: Mon Aug 25 16:22:43 2008
New Revision: 688918
URL: http://svn.apache.org/viewvc?rev=688918&view=rev
Log:
PIG-392: error handling with multiple MR jobs
Modified:
incubator/pig/branches/types/CHANGES.txt
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
Modified: incubator/pig/branches/types/CHANGES.txt
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/CHANGES.txt?rev=688918&r1=688917&r2=688918&view=diff
==============================================================================
--- incubator/pig/branches/types/CHANGES.txt (original)
+++ incubator/pig/branches/types/CHANGES.txt Mon Aug 25 16:22:43 2008
@@ -171,3 +171,5 @@
PIG-329: TestStoreOld, 2 unit tests were broken
PIG-353: parsing of complex types
+
+ PIG-392: error handling with multiple MRjobs
Modified:
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java?rev=688918&r1=688917&r2=688918&view=diff
==============================================================================
---
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java
(original)
+++
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/LocalLauncher.java
Mon Aug 25 16:22:43 2008
@@ -64,18 +64,17 @@
log.info((int)(prog * 100) + "% complete");
lastProg = prog;
}
- lastProg = calculateProgress(jc, jobClient)/numMRJobs;
- if(isComplete(lastProg))
- log.info("Completed Successfully");
- else{
- log.info("Unsuccessful attempt. Completed " + lastProg * 100 + "%
of the job");
- List<Job> failedJobs = jc.getFailedJobs();
- if(failedJobs==null)
- throw new ExecException("Something terribly wrong with Job
Control.");
- for (Job job : failedJobs) {
- getStats(job,jobClient);
+ // Look to see if any jobs failed. If so, we need to report that.
+ List<Job> failedJobs = jc.getFailedJobs();
+ if (failedJobs != null && failedJobs.size() > 0) {
+ log.error("Map reduce job failed");
+ for (Job fj : failedJobs) {
+ log.error(fj.getMessage());
+ getStats(fj, jobClient);
}
+ return false;
}
+
List<Job> succJobs = jc.getSuccessfulJobs();
if(succJobs!=null)
for(Job job : succJobs){
@@ -84,7 +83,7 @@
jc.stop();
- return isComplete(lastProg);
+ return true;
}
@Override
Modified:
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java?rev=688918&r1=688917&r2=688918&view=diff
==============================================================================
---
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
(original)
+++
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
Mon Aug 25 16:22:43 2008
@@ -81,18 +81,17 @@
log.info((int)(prog * 100) + "% complete");
lastProg = prog;
}
- lastProg = calculateProgress(jc, jobClient)/numMRJobs;
- if(isComplete(lastProg))
- log.info("Completed Successfully");
- else{
- log.info("Unsuccessful attempt. Completed " + lastProg * 100 + "%
of the job");
- List<Job> failedJobs = jc.getFailedJobs();
- if(failedJobs==null)
- throw new ExecException("Something terribly wrong with Job
Control.");
- for (Job job : failedJobs) {
- getStats(job,jobClient);
+ // Look to see if any jobs failed. If so, we need to report that.
+ List<Job> failedJobs = jc.getFailedJobs();
+ if (failedJobs != null && failedJobs.size() > 0) {
+ log.error("Map reduce job failed");
+ for (Job fj : failedJobs) {
+ log.error(fj.getMessage());
+ getStats(fj, jobClient);
}
+ return false;
}
+
List<Job> succJobs = jc.getSuccessfulJobs();
if(succJobs!=null)
for(Job job : succJobs){
@@ -101,7 +100,7 @@
jc.stop();
- return isComplete(lastProg);
+ return true;
}
@Override
