Author: jlowe Date: Wed Oct 9 16:53:52 2013 New Revision: 1530710 URL: http://svn.apache.org/r1530710 Log: MAPREDUCE-5569. FloatSplitter is not generating correct splits. Contributed by Nathan Roberts
Added: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/lib/db/TestSplitters.java (with props) Modified: hadoop/common/branches/branch-1/CHANGES.txt hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/lib/db/FloatSplitter.java Modified: hadoop/common/branches/branch-1/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1530710&r1=1530709&r2=1530710&view=diff ============================================================================== --- hadoop/common/branches/branch-1/CHANGES.txt (original) +++ hadoop/common/branches/branch-1/CHANGES.txt Wed Oct 9 16:53:52 2013 @@ -148,6 +148,9 @@ Release 1.3.0 - unreleased HADOOP-10009. Backport HADOOP-7808 to branch-1: fix NPE in SecurityUtil::setTokenService(). (Haohui Mai via jing9) + MAPREDUCE-5569. FloatSplitter is not generating correct splits (Nathan + Roberts via jlowe) + Release 1.2.2 - unreleased INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/lib/db/FloatSplitter.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/lib/db/FloatSplitter.java?rev=1530710&r1=1530709&r2=1530710&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/lib/db/FloatSplitter.java (original) +++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/lib/db/FloatSplitter.java Wed Oct 9 16:53:52 2013 @@ -89,7 +89,7 @@ public class FloatSplitter implements DB // Catch any overage and create the closed interval for the last split. if (curLower <= maxVal || splits.size() == 1) { splits.add(new DataDrivenDBInputFormat.DataDrivenDBInputSplit( - lowClausePrefix + Double.toString(curUpper), + lowClausePrefix + Double.toString(curLower), colName + " <= " + Double.toString(maxVal))); } Added: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/lib/db/TestSplitters.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/lib/db/TestSplitters.java?rev=1530710&view=auto ============================================================================== --- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/lib/db/TestSplitters.java (added) +++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/lib/db/TestSplitters.java Wed Oct 9 16:53:52 2013 @@ -0,0 +1,163 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.lib.db; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.math.BigDecimal; +import java.sql.ResultSet; +import java.util.List; +import java.util.regex.Pattern; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.lib.db.DataDrivenDBInputFormat.DataDrivenDBInputSplit; +import org.junit.Before; +import org.junit.Test; + +/** + * Test Splitters. Splitters should build parts of sql sentences for split result. + */ +public class TestSplitters { + + private Configuration configuration; + + @Before + public void setup() { + configuration = new Configuration(); + configuration.setInt("mapred.map.tasks", 2); + } + + @Test(timeout=2000) + public void testBooleanSplitter() throws Exception{ + BooleanSplitter splitter = new BooleanSplitter(); + ResultSet result = mock(ResultSet.class); + when(result.getString(1)).thenReturn("result1"); + + List<InputSplit> splits=splitter.split(configuration, result, "column"); + assertSplits(new String[] {"column = FALSE column = FALSE", + "column IS NULL column IS NULL"}, splits); + + when(result.getString(1)).thenReturn("result1"); + when(result.getString(2)).thenReturn("result2"); + when(result.getBoolean(1)).thenReturn(true); + when(result.getBoolean(2)).thenReturn(false); + + splits=splitter.split(configuration, result, "column"); + assertEquals(0, splits.size()); + + when(result.getString(1)).thenReturn("result1"); + when(result.getString(2)).thenReturn("result2"); + when(result.getBoolean(1)).thenReturn(false); + when(result.getBoolean(2)).thenReturn(true); + + splits = splitter.split(configuration, result, "column"); + assertSplits(new String[] { + "column = FALSE column = FALSE", ".*column = TRUE"}, splits); + } + + @Test(timeout=2000) + public void testFloatSplitter() throws Exception{ + FloatSplitter splitter = new FloatSplitter(); + + ResultSet results = mock(ResultSet.class); + + List<InputSplit> splits = splitter.split(configuration, results, "column"); + assertSplits(new String[] {".*column IS NULL"}, splits); + + when(results.getString(1)).thenReturn("result1"); + when(results.getString(2)).thenReturn("result2"); + when(results.getDouble(1)).thenReturn(5.0); + when(results.getDouble(2)).thenReturn(7.0); + + splits = splitter.split(configuration, results, "column1"); + assertSplits(new String[] {"column1 >= 5.0 column1 < 6.0", + "column1 >= 6.0 column1 <= 7.0"}, splits); + } + + @Test(timeout=2000) + public void testBigDecimalSplitter() throws Exception{ + BigDecimalSplitter splitter = new BigDecimalSplitter(); + ResultSet result = mock(ResultSet.class); + + List<InputSplit> splits = splitter.split(configuration, result, "column"); + assertSplits(new String[] {".*column IS NULL"}, splits); + + when(result.getString(1)).thenReturn("result1"); + when(result.getString(2)).thenReturn("result2"); + when(result.getBigDecimal(1)).thenReturn(new BigDecimal(10)); + when(result.getBigDecimal(2)).thenReturn(new BigDecimal(12)); + + splits = splitter.split(configuration, result, "column1"); + assertSplits(new String[] {"column1 >= 10 column1 < 11", + "column1 >= 11 column1 <= 12"}, splits); + } + + @Test(timeout=2000) + public void testIntegerSplitter() throws Exception{ + IntegerSplitter splitter = new IntegerSplitter(); + ResultSet result = mock(ResultSet.class); + + List<InputSplit> splits = splitter.split(configuration, result, "column"); + assertSplits(new String[] {".*column IS NULL"}, splits); + + when(result.getString(1)).thenReturn("result1"); + when(result.getString(2)).thenReturn("result2"); + when(result.getLong(1)).thenReturn(8L); + when(result.getLong(2)).thenReturn(19L); + + splits = splitter.split(configuration, result, "column1"); + assertSplits(new String[] {"column1 >= 8 column1 < 13", + "column1 >= 13 column1 < 18", "column1 >= 18 column1 <= 19"}, splits); + } + + @Test(timeout=2000) + public void testTextSplitter() throws Exception{ + TextSplitter splitter = new TextSplitter(); + ResultSet result = mock(ResultSet.class); + + List<InputSplit> splits = splitter.split(configuration, result, "column"); + assertSplits(new String[] {"column IS NULL column IS NULL"}, splits); + + when(result.getString(1)).thenReturn("result1"); + when(result.getString(2)).thenReturn("result2"); + + splits = splitter.split(configuration, result, "column1"); + assertSplits(new String[] {"column1 >= 'result1' column1 < 'result1.'", + "column1 >= 'result1' column1 <= 'result2'"}, splits); + } + + private void assertSplits(String[] expectedSplitRE, + List<InputSplit> splits) throws IOException { + assertEquals(expectedSplitRE.length, splits.size()); + for (int i = 0; i < expectedSplitRE.length; i++) { + DataDrivenDBInputSplit split = (DataDrivenDBInputSplit) splits.get(i); + String actualExpr = split.getLowerClause() + " " + split.getUpperClause(); + assertTrue("Split #" + (i+1) + " expression is wrong." + + " Expected " + expectedSplitRE[i] + + " Actual " + actualExpr, + Pattern.matches(expectedSplitRE[i], actualExpr)); + } + } + +} Propchange: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/lib/db/TestSplitters.java ------------------------------------------------------------------------------ svn:eol-style = native