HIVE-10781 : HadoopJobExecHelper Leaks RunningJobs (Chinna R Lalam via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <hashut...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c75167d6 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c75167d6 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c75167d6 Branch: refs/heads/llap Commit: c75167d6e44e9c50f97e0ac26646a48f92cf36e8 Parents: 1f75e34 Author: Chinna R Lalam <chinna.r.la...@intel.com> Authored: Thu May 21 02:29:00 2015 -0700 Committer: Ashutosh Chauhan <hashut...@apache.org> Committed: Mon May 25 23:10:30 2015 -0700 ---------------------------------------------------------------------- ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java | 4 +++- .../java/org/apache/hadoop/hive/ql/io/merge/MergeFileTask.java | 6 ++++-- .../apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java | 4 +++- .../hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java | 4 +++- 4 files changed, 13 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c75167d6/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java index d7a08ec..e4f9543 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java @@ -462,10 +462,12 @@ public class ExecDriver extends Task<MapredWork> implements Serializable, Hadoop if (returnVal != 0) { rj.killJob(); } - HadoopJobExecHelper.runningJobs.remove(rj); jobID = rj.getID().toString(); } } catch (Exception e) { + LOG.warn(e); + } finally { + HadoopJobExecHelper.runningJobs.remove(rj); } } http://git-wip-us.apache.org/repos/asf/hive/blob/c75167d6/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileTask.java b/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileTask.java index 08ab1c9..bd50b46 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileTask.java @@ -176,7 +176,6 @@ public class MergeFileTask extends Task<MergeFileWork> implements Serializable, if (returnVal != 0) { rj.killJob(); } - HadoopJobExecHelper.runningJobs.remove(rj); jobID = rj.getID().toString(); } // get the list of Dynamic partition paths @@ -189,7 +188,8 @@ public class MergeFileTask extends Task<MergeFileWork> implements Serializable, } } } catch (Exception e) { - // jobClose needs to execute successfully otherwise fail task + // jobClose needs to execute successfully otherwise fail task + LOG.warn(e); if (success) { success = false; returnVal = 3; @@ -198,6 +198,8 @@ public class MergeFileTask extends Task<MergeFileWork> implements Serializable, console.printError(mesg, "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e)); } + } finally { + HadoopJobExecHelper.runningJobs.remove(rj); } } http://git-wip-us.apache.org/repos/asf/hive/blob/c75167d6/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java b/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java index f4ffdd6..3cb9e9c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java @@ -245,10 +245,12 @@ public class PartialScanTask extends Task<PartialScanWork> implements if (returnVal != 0) { rj.killJob(); } - HadoopJobExecHelper.runningJobs.remove(rj); jobID = rj.getID().toString(); } } catch (Exception e) { + LOG.warn(e); + } finally { + HadoopJobExecHelper.runningJobs.remove(rj); } } http://git-wip-us.apache.org/repos/asf/hive/blob/c75167d6/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java b/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java index 4633820..149ad93 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java @@ -213,12 +213,14 @@ public class ColumnTruncateTask extends Task<ColumnTruncateWork> implements Seri if (returnVal != 0) { rj.killJob(); } - HadoopJobExecHelper.runningJobs.remove(rj); jobID = rj.getID().toString(); } ColumnTruncateMapper.jobClose(outputPath, success, job, console, work.getDynPartCtx(), null); } catch (Exception e) { + LOG.warn(e); + } finally { + HadoopJobExecHelper.runningJobs.remove(rj); } }
