svn commit: r1603670 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ hadoop-mapreduce-client/hadoop-
Author: kasha
Date: Wed Jun 18 23:28:50 2014
New Revision: 1603670
URL: http://svn.apache.org/r1603670
Log:
MAPREDUCE-5896. InputSplits should indicate which locations have the block
cached in memory. (Sandy Ryza via kasha)
Added:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/InputSplitWithLocationInfo.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/SplitLocationInfo.java
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileSplit.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/InputSplit.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileSplit.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestFileInputFormat.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1603670r1=1603669r2=1603670view=diff
==
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Wed Jun 18
23:28:50 2014
@@ -213,6 +213,9 @@ Release 2.5.0 - UNRELEASED
MAPREDUCE-5834. Increased test-timeouts in TestGridMixClasses to avoid
occassional failures. (Mit Desai via vinodkv)
+MAPREDUCE-5896. InputSplits should indicate which locations have the block
+cached in memory. (Sandy Ryza via kasha)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java?rev=1603670r1=1603669r2=1603670view=diff
==
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
Wed Jun 18 23:28:50 2014
@@ -295,6 +295,15 @@ public abstract class FileInputFormatK,
String[] hosts) {
return new FileSplit(file, start, length, hosts);
}
+
+ /**
+ * A factory that makes the split for this class. It can be overridden
+ * by sub-classes to make sub-types
+ */
+ protected FileSplit makeSplit(Path file, long start, long length,
+String[] hosts, String[] inMemoryHosts) {
+return new FileSplit(file, start, length, hosts, inMemoryHosts);
+ }
/** Splits files returned by {@link #listStatus(JobConf)} when
* they're too big.*/
@@ -337,22 +346,22 @@ public abstract class FileInputFormatK,
long bytesRemaining = length;
while (((double) bytesRemaining)/splitSize SPLIT_SLOP) {
-String[] splitHosts = getSplitHosts(blkLocations,
+String[][] splitHosts = getSplitHostsAndCachedHosts(blkLocations,
length-bytesRemaining, splitSize, clusterMap);
splits.add(makeSplit(path, length-bytesRemaining, splitSize,
-splitHosts));
+splitHosts[0], splitHosts[1]));
bytesRemaining -= splitSize;
}
if (bytesRemaining != 0) {
-String[] splitHosts = getSplitHosts(blkLocations, length
+String[][] splitHosts = getSplitHostsAndCachedHosts(blkLocations,
length
- bytesRemaining, bytesRemaining, clusterMap);
splits.add(makeSplit(path, length - bytesRemaining, bytesRemaining,
-
svn commit: r1603694 - in /hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/ hadoop-mapreduce-client/hado
Author: tgraves Date: Thu Jun 19 01:17:43 2014 New Revision: 1603694 URL: http://svn.apache.org/r1603694 Log: Preparing for release 0.23.11 Modified: hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/pom.xml Modified: hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/CHANGES.txt?rev=1603694r1=1603693r2=1603694view=diff == --- hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/CHANGES.txt Thu Jun 19 01:17:43 2014 @@ -1,6 +1,6 @@ Hadoop MapReduce Change Log -Release 0.23.11 - UNRELEASED +Release 0.23.11 - 2014-06-26 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml?rev=1603694r1=1603693r2=1603694view=diff == --- hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml (original) +++ hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml Thu Jun 19 01:17:43 2014 @@ -19,12 +19,12 @@ parent artifactIdhadoop-mapreduce-client/artifactId groupIdorg.apache.hadoop/groupId -version0.23.11-SNAPSHOT/version +version0.23.11/version /parent modelVersion4.0.0/modelVersion groupIdorg.apache.hadoop/groupId artifactIdhadoop-mapreduce-client-app/artifactId - version0.23.11-SNAPSHOT/version + version0.23.11/version namehadoop-mapreduce-client-app/name properties Modified: hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml?rev=1603694r1=1603693r2=1603694view=diff == --- hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml (original) +++ hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml Thu Jun 19 01:17:43 2014 @@ -19,12 +19,12 @@ parent artifactIdhadoop-mapreduce-client/artifactId groupIdorg.apache.hadoop/groupId -version0.23.11-SNAPSHOT/version +version0.23.11/version /parent modelVersion4.0.0/modelVersion groupIdorg.apache.hadoop/groupId artifactIdhadoop-mapreduce-client-common/artifactId - version0.23.11-SNAPSHOT/version + version0.23.11/version namehadoop-mapreduce-client-common/name properties Modified: hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml?rev=1603694r1=1603693r2=1603694view=diff == --- hadoop/common/branches/branch-0.23.11/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml (original) +++
