MAPREDUCE-6793. io.sort.factor code default and mapred-default.xml values inconsistent. Contributed by Gera Shegalov.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/683e0c71 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/683e0c71 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/683e0c71 Branch: refs/heads/YARN-2915 Commit: 683e0c71fe09600a24bdd7b707a613fe70ff1f6e Parents: 009452b Author: Rohith Sharma K S <rohithsharm...@apache.org> Authored: Mon Nov 21 22:10:57 2016 +0530 Committer: Rohith Sharma K S <rohithsharm...@apache.org> Committed: Mon Nov 21 22:10:57 2016 +0530 ---------------------------------------------------------------------- .../src/main/java/org/apache/hadoop/mapred/MapTask.java | 6 ++++-- .../main/java/org/apache/hadoop/mapreduce/MRJobConfig.java | 4 ++++ .../apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java | 3 ++- .../apache/hadoop/mapreduce/task/reduce/TestMergeManager.java | 7 +++++++ 4 files changed, 17 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/683e0c71/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java index 45431e6..3753fba 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java @@ -967,7 +967,8 @@ public class MapTask extends Task { //sanity checks final float spillper = job.getFloat(JobContext.MAP_SORT_SPILL_PERCENT, (float)0.8); - final int sortmb = job.getInt(JobContext.IO_SORT_MB, 100); + final int sortmb = job.getInt(MRJobConfig.IO_SORT_MB, + MRJobConfig.DEFAULT_IO_SORT_MB); indexCacheMemoryLimit = job.getInt(JobContext.INDEX_CACHE_MEMORY_LIMIT, INDEX_CACHE_MEMORY_LIMIT_DEFAULT); if (spillper > (float)1.0 || spillper <= (float)0.0) { @@ -1920,7 +1921,8 @@ public class MapTask extends Task { } } - int mergeFactor = job.getInt(JobContext.IO_SORT_FACTOR, 100); + int mergeFactor = job.getInt(MRJobConfig.IO_SORT_FACTOR, + MRJobConfig.DEFAULT_IO_SORT_FACTOR); // sort the segments only if there are intermediate merges boolean sortSegments = segmentList.size() > mergeFactor; //merge http://git-wip-us.apache.org/repos/asf/hadoop/blob/683e0c71/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 1325b74..2ad86ef 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -215,8 +215,12 @@ public interface MRJobConfig { public static final String IO_SORT_FACTOR = "mapreduce.task.io.sort.factor"; + public static final int DEFAULT_IO_SORT_FACTOR = 10; + public static final String IO_SORT_MB = "mapreduce.task.io.sort.mb"; + public static final int DEFAULT_IO_SORT_MB = 100; + public static final String INDEX_CACHE_MEMORY_LIMIT = "mapreduce.task.index.cache.limit.bytes"; public static final String PRESERVE_FAILED_TASK_FILES = "mapreduce.task.files.preserve.failedtasks"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/683e0c71/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java index 09fe0cb..f26c10a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java @@ -175,7 +175,8 @@ public class MergeManagerImpl<K, V> implements MergeManager<K, V> { MRJobConfig.REDUCE_MEMORY_TOTAL_BYTES, Runtime.getRuntime().maxMemory()) * maxInMemCopyUse); - this.ioSortFactor = jobConf.getInt(MRJobConfig.IO_SORT_FACTOR, 100); + this.ioSortFactor = jobConf.getInt(MRJobConfig.IO_SORT_FACTOR, + MRJobConfig.DEFAULT_IO_SORT_FACTOR); final float singleShuffleMemoryLimitPercent = jobConf.getFloat(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT, http://git-wip-us.apache.org/repos/asf/hadoop/blob/683e0c71/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java index 325d2f9..1e0dddd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java @@ -207,6 +207,13 @@ public class TestMergeManager { } } + @Test + public void testIoSortDefaults() { + final JobConf jobConf = new JobConf(); + assertEquals(10, jobConf.getInt(MRJobConfig.IO_SORT_FACTOR, 100)); + assertEquals(100, jobConf.getInt(MRJobConfig.IO_SORT_MB, 10)); + } + @SuppressWarnings({ "unchecked", "deprecation" }) @Test(timeout=10000) public void testOnDiskMerger() throws IOException, URISyntaxException, --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
