svn commit: r1203902 - in /hadoop/common/branches/branch-0.20-security: ./ src/mapred/org/apache/hadoop/filecache/ src/mapred/org/apache/hadoop/mapred/ src/test/org/apache/hadoop/filecache/

[eli]

Author: eli
Date: Fri Nov 18 23:43:37 2011
New Revision: 1203902

URL: http://svn.apache.org/viewvc?rev=1203902&view=rev
Log:
MAPREDUCE-3343. TaskTracker Out of Memory because of distributed cache. 
Contributed by Zhao Yunjiong

Modified:
    hadoop/common/branches/branch-0.20-security/CHANGES.txt
    
hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
    
hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
    
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java

Modified: hadoop/common/branches/branch-0.20-security/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/CHANGES.txt?rev=1203902&r1=1203901&r2=1203902&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20-security/CHANGES.txt Fri Nov 18 23:43:37 
2011
@@ -35,6 +35,9 @@ Release 0.20.206.0 - unreleased
     MAPREDUCE-2073. TestTrackerDistributedCacheManager should be
     up-front about requirements on build environment. (todd)
 
+    MAPREDUCE-3343. TaskTracker Out of Memory because of distributed cache.
+    (Zhao Yunjiong).
+
   IMPROVEMENTS
 
     MAPREDUCE-2836. Provide option to fail jobs when submitted to

Modified: 
hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java?rev=1203902&r1=1203901&r2=1203902&view=diff
==============================================================================
--- 
hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
 (original)
+++ 
hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
 Fri Nov 18 23:43:37 2011
@@ -675,6 +675,18 @@ public class TrackerDistributedCacheMana
     }
   }
 
+  public void removeTaskDistributedCacheManager(JobID jobId) {
+    jobArchives.remove(jobId);
+  }
+
+  /*
+   * This method is called from unit tests.
+   */
+  protected TaskDistributedCacheManager getTaskDistributedCacheManager(
+      JobID jobId) {
+    return jobArchives.get(jobId);
+  }
+
   /**
    * Determines timestamps of files to be cached, and stores those
    * in the configuration.  This is intended to be used internally by JobClient

Modified: 
hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/mapred/TaskTracker.java?rev=1203902&r1=1203901&r2=1203902&view=diff
==============================================================================
--- 
hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
 (original)
+++ 
hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
 Fri Nov 18 23:43:37 2011
@@ -2081,8 +2081,9 @@ public class TaskTracker implements MRCo
       runningJobs.remove(jobId);
     }
     getJobTokenSecretManager().removeTokenForJob(jobId.toString());  
-  }      
-    
+    distributedCacheManager.removeTaskDistributedCacheManager(jobId);
+  }
+
   /**
    * This job's files are no longer needed on this TT, remove them.
    *

Modified: 
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java?rev=1203902&r1=1203901&r2=1203902&view=diff
==============================================================================
--- 
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java
 (original)
+++ 
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java
 Fri Nov 18 23:43:37 2011
@@ -1147,4 +1147,23 @@ public class TestTrackerDistributedCache
     }
   }
 
+  public void testRemoveTaskDistributedCacheManager() throws Exception {
+    if (!canRun()) {
+      return;
+    }
+    TrackerDistributedCacheManager manager = new 
TrackerDistributedCacheManager(
+        conf, taskController);
+    JobID jobId = new JobID("jobtracker", 1);
+    manager.newTaskDistributedCacheManager(jobId, conf);
+
+    TaskDistributedCacheManager taskDistributedCacheManager = manager
+        .getTaskDistributedCacheManager(jobId);
+    assertNotNull(taskDistributedCacheManager);
+
+    manager.removeTaskDistributedCacheManager(jobId);
+
+    taskDistributedCacheManager = 
manager.getTaskDistributedCacheManager(jobId);
+    assertNull(taskDistributedCacheManager);
+  }
+
 }