Repository: giraph Updated Branches: refs/heads/trunk 77ae12e00 -> c9ab310db
[GIRAPH-1053] Log exceptions to command line Summary: When we know an exception occurred, log it to command line to make it easier for people running jobs to see what the issue was. Test Plan: Ran two jobs, one with error in input one with error in compute, verified exception is printed to command line. Also ran a normal job and verified it didn't print anything new to command line Differential Revision: https://reviews.facebook.net/D56931 Project: http://git-wip-us.apache.org/repos/asf/giraph/repo Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/c9ab310d Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/c9ab310d Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/c9ab310d Branch: refs/heads/trunk Commit: c9ab310db6c63e9151e0cbd01b83f3b2378b4676 Parents: 77ae12e Author: Maja Kabiljo <[email protected]> Authored: Mon Apr 18 17:34:36 2016 -0700 Committer: Maja Kabiljo <[email protected]> Committed: Mon Apr 18 18:53:01 2016 -0700 ---------------------------------------------------------------------- .../main/java/org/apache/giraph/graph/GraphMapper.java | 5 +++++ .../apache/giraph/graph/JobProgressTrackerClientNoOp.java | 4 ++++ .../giraph/graph/RetryableJobProgressTrackerClient.java | 10 ++++++++++ .../java/org/apache/giraph/job/JobProgressTracker.java | 9 +++++++++ .../org/apache/giraph/job/JobProgressTrackerService.java | 5 +++++ 5 files changed, 33 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/giraph/blob/c9ab310d/giraph-core/src/main/java/org/apache/giraph/graph/GraphMapper.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/GraphMapper.java b/giraph-core/src/main/java/org/apache/giraph/graph/GraphMapper.java index 864ae9a..952249a 100644 --- a/giraph-core/src/main/java/org/apache/giraph/graph/GraphMapper.java +++ b/giraph-core/src/main/java/org/apache/giraph/graph/GraphMapper.java @@ -18,6 +18,7 @@ package org.apache.giraph.graph; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.hadoop.filecache.DistributedCache; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; @@ -97,6 +98,10 @@ public class GraphMapper<I extends WritableComparable, V extends Writable, } catch (RuntimeException e) { // CHECKSTYLE: resume IllegalCatch LOG.error("Caught an unrecoverable exception " + e.getMessage(), e); + graphTaskManager.getJobProgressTracker().logError( + "Exception occurred on mapper " + + graphTaskManager.getConf().getTaskPartition() + ": " + + ExceptionUtils.getStackTrace(e)); graphTaskManager.zooKeeperCleanup(); graphTaskManager.workerFailureCleanup(); throw new IllegalStateException( http://git-wip-us.apache.org/repos/asf/giraph/blob/c9ab310d/giraph-core/src/main/java/org/apache/giraph/graph/JobProgressTrackerClientNoOp.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/JobProgressTrackerClientNoOp.java b/giraph-core/src/main/java/org/apache/giraph/graph/JobProgressTrackerClientNoOp.java index d75fd42..369941f 100644 --- a/giraph-core/src/main/java/org/apache/giraph/graph/JobProgressTrackerClientNoOp.java +++ b/giraph-core/src/main/java/org/apache/giraph/graph/JobProgressTrackerClientNoOp.java @@ -38,6 +38,10 @@ public class JobProgressTrackerClientNoOp implements JobProgressTrackerClient { } @Override + public void logError(String logLine) { + } + + @Override public void logFailure(String reason) { } http://git-wip-us.apache.org/repos/asf/giraph/blob/c9ab310d/giraph-core/src/main/java/org/apache/giraph/graph/RetryableJobProgressTrackerClient.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/RetryableJobProgressTrackerClient.java b/giraph-core/src/main/java/org/apache/giraph/graph/RetryableJobProgressTrackerClient.java index b4915f1..9ce12ed 100644 --- a/giraph-core/src/main/java/org/apache/giraph/graph/RetryableJobProgressTrackerClient.java +++ b/giraph-core/src/main/java/org/apache/giraph/graph/RetryableJobProgressTrackerClient.java @@ -121,6 +121,16 @@ public class RetryableJobProgressTrackerClient } @Override + public synchronized void logError(final String logLine) { + executeWithRetry(new Runnable() { + @Override + public void run() { + jobProgressTracker.logError(logLine); + } + }); + } + + @Override public synchronized void logFailure(final String reason) { executeWithRetry(new Runnable() { @Override http://git-wip-us.apache.org/repos/asf/giraph/blob/c9ab310d/giraph-core/src/main/java/org/apache/giraph/job/JobProgressTracker.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/job/JobProgressTracker.java b/giraph-core/src/main/java/org/apache/giraph/job/JobProgressTracker.java index 95bc56d..4da5450 100644 --- a/giraph-core/src/main/java/org/apache/giraph/job/JobProgressTracker.java +++ b/giraph-core/src/main/java/org/apache/giraph/job/JobProgressTracker.java @@ -52,6 +52,15 @@ public interface JobProgressTracker { void logInfo(String logLine); /** + * Call this when you want to log an error line from any mapper to command + * line + * + * @param logLine Line to log + */ + @ThriftMethod + void logError(String logLine); + + /** * Notify that job is failing * * @param reason Reason for failure http://git-wip-us.apache.org/repos/asf/giraph/blob/c9ab310d/giraph-core/src/main/java/org/apache/giraph/job/JobProgressTrackerService.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/job/JobProgressTrackerService.java b/giraph-core/src/main/java/org/apache/giraph/job/JobProgressTrackerService.java index 4898cfa..b08bf3e 100644 --- a/giraph-core/src/main/java/org/apache/giraph/job/JobProgressTrackerService.java +++ b/giraph-core/src/main/java/org/apache/giraph/job/JobProgressTrackerService.java @@ -184,6 +184,11 @@ public class JobProgressTrackerService implements JobProgressTracker { } @Override + public void logError(String logLine) { + LOG.error(logLine); + } + + @Override public void logFailure(String reason) { LOG.fatal(reason); finished = true;
