Author: shv Date: Fri Feb 4 22:00:38 2011 New Revision: 1067305 URL: http://svn.apache.org/viewvc?rev=1067305&view=rev Log: MAPREDUCE-1915. Fix IndexOutOfBoundsException in IndexCache. Contributed by Priyo Mustafi.
Modified: hadoop/mapreduce/trunk/CHANGES.txt hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/IndexCache.java hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestIndexCache.java Modified: hadoop/mapreduce/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1067305&r1=1067304&r2=1067305&view=diff ============================================================================== --- hadoop/mapreduce/trunk/CHANGES.txt (original) +++ hadoop/mapreduce/trunk/CHANGES.txt Fri Feb 4 22:00:38 2011 @@ -538,6 +538,9 @@ Release 0.22.0 - Unreleased MAPREDUCE-2188. The new API MultithreadedMapper doesn't initialize RecordReader. (Owen O'Malley via shv) + MAPREDUCE-1915. Fix IndexOutOfBoundsException in IndexCache. + (Priyo Mustafi via shv) + Release 0.21.1 - Unreleased NEW FEATURES Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/IndexCache.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/IndexCache.java?rev=1067305&r1=1067304&r2=1067305&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/IndexCache.java (original) +++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/IndexCache.java Fri Feb 4 22:00:38 2011 @@ -80,7 +80,7 @@ class IndexCache { } if (info.mapSpillRecord.size() == 0 || - info.mapSpillRecord.size() < reduce) { + info.mapSpillRecord.size() <= reduce) { throw new IOException("Invalid request " + " Map Id = " + mapId + " Reducer = " + reduce + " Index Info Length = " + info.mapSpillRecord.size()); Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestIndexCache.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestIndexCache.java?rev=1067305&r1=1067304&r2=1067305&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestIndexCache.java (original) +++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestIndexCache.java Fri Feb 4 22:00:38 2011 @@ -149,6 +149,50 @@ public class TestIndexCache extends Test } } + public void testInvalidReduceNumberOrLength() throws Exception { + JobConf conf = new JobConf(); + FileSystem fs = FileSystem.getLocal(conf).getRaw(); + Path p = new Path(System.getProperty("test.build.data", "/tmp"), + "cache").makeQualified(fs); + fs.delete(p, true); + conf.setInt(TTConfig.TT_INDEX_CACHE, 1); + final int partsPerMap = 1000; + final int bytesPerFile = partsPerMap * 24; + IndexCache cache = new IndexCache(conf); + + // fill cache + Path feq = new Path(p, "invalidReduceOrPartsPerMap"); + writeFile(fs, feq, bytesPerFile, partsPerMap); + + // Number of reducers should always be less than partsPerMap as reducer + // numbers start from 0 and there cannot be more reducer than parts + + try { + // Number of reducers equal to partsPerMap + cache.getIndexInformation("reduceEqualPartsPerMap", + partsPerMap, // reduce number == partsPerMap + feq, UserGroupInformation.getCurrentUser().getShortUserName()); + fail("Number of reducers equal to partsPerMap did not fail"); + } catch (Exception e) { + if (!(e instanceof IOException)) { + throw e; + } + } + + try { + // Number of reducers more than partsPerMap + cache.getIndexInformation( + "reduceMorePartsPerMap", + partsPerMap + 1, // reduce number > partsPerMap + feq, UserGroupInformation.getCurrentUser().getShortUserName()); + fail("Number of reducers more than partsPerMap did not fail"); + } catch (Exception e) { + if (!(e instanceof IOException)) { + throw e; + } + } + } + private static void checkRecord(IndexRecord rec, long fill) { assertEquals(fill, rec.startOffset); assertEquals(fill, rec.rawLength);