Author: jlowe
Date: Mon Jun 3 14:51:37 2013
New Revision: 1489015
URL: http://svn.apache.org/r1489015
Log:
svn merge -c 1489012 FIXES: MAPREDUCE-5268. Improve history server startup
performance. Contributed by Karthik Kambatla
Added:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobIdHistoryFileInfoMap.java
- copied unchanged from r1489012,
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobIdHistoryFileInfoMap.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobListCache.java
- copied unchanged from r1489012,
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobListCache.java
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1489015&r1=1489014&r2=1489015&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Mon
Jun 3 14:51:37 2013
@@ -144,6 +144,9 @@ Release 2.1.0-beta - UNRELEASED
MAPREDUCE-4974. Optimising the LineRecordReader initialize() method
(Gelesh via bobby)
+ MAPREDUCE-5268. Improve history server startup performance (Karthik
+ Kambatla via jlowe)
+
BUG FIXES
MAPREDUCE-4671. AM does not tell the RM about container requests which are
@@ -936,6 +939,21 @@ Release 2.0.0-alpha - 05-23-2012
MAPREDUCE-4444. nodemanager fails to start when one of the local-dirs is
bad (Jason Lowe via bobby)
+Release 0.23.9 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ MAPREDUCE-5268. Improve history server startup performance (Karthik
+ Kambatla via jlowe)
+
+ BUG FIXES
+
Release 0.23.8 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java?rev=1489015&r1=1489014&r2=1489015&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
Mon Jun 3 14:51:37 2013
@@ -26,6 +26,7 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.NavigableSet;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
@@ -36,6 +37,7 @@ import java.util.concurrent.LinkedBlocki
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -131,19 +133,73 @@ public class HistoryFileManager extends
}
}
- static class JobListCache {
+ /**
+ * Wrapper around {@link ConcurrentSkipListMap} that maintains size along
+ * side for O(1) size() implementation for use in JobListCache.
+ *
+ * Note: The size is not updated atomically with changes additions/removals.
+ * This race can lead to size() returning an incorrect size at times.
+ */
+ static class JobIdHistoryFileInfoMap {
private ConcurrentSkipListMap<JobId, HistoryFileInfo> cache;
+ private AtomicInteger mapSize;
+
+ JobIdHistoryFileInfoMap() {
+ cache = new ConcurrentSkipListMap<JobId, HistoryFileInfo>();
+ mapSize = new AtomicInteger();
+ }
+
+ public HistoryFileInfo putIfAbsent(JobId key, HistoryFileInfo value) {
+ HistoryFileInfo ret = cache.putIfAbsent(key, value);
+ if (ret == null) {
+ mapSize.incrementAndGet();
+ }
+ return ret;
+ }
+
+ public HistoryFileInfo remove(JobId key) {
+ HistoryFileInfo ret = cache.remove(key);
+ if (ret != null) {
+ mapSize.decrementAndGet();
+ }
+ return ret;
+ }
+
+ /**
+ * Returns the recorded size of the internal map. Note that this could be
out
+ * of sync with the actual size of the map
+ * @return "recorded" size
+ */
+ public int size() {
+ return mapSize.get();
+ }
+
+ public HistoryFileInfo get(JobId key) {
+ return cache.get(key);
+ }
+
+ public NavigableSet<JobId> navigableKeySet() {
+ return cache.navigableKeySet();
+ }
+
+ public Collection<HistoryFileInfo> values() {
+ return cache.values();
+ }
+ }
+
+ static class JobListCache {
+ private JobIdHistoryFileInfoMap cache;
private int maxSize;
private long maxAge;
public JobListCache(int maxSize, long maxAge) {
this.maxSize = maxSize;
this.maxAge = maxAge;
- this.cache = new ConcurrentSkipListMap<JobId, HistoryFileInfo>();
+ this.cache = new JobIdHistoryFileInfoMap();
}
public HistoryFileInfo addIfAbsent(HistoryFileInfo fileInfo) {
- JobId jobId = fileInfo.getJobIndexInfo().getJobId();
+ JobId jobId = fileInfo.getJobId();
if (LOG.isDebugEnabled()) {
LOG.debug("Adding " + jobId + " to job list cache with "
+ fileInfo.getJobIndexInfo());
