Repository: incubator-hawq Updated Branches: refs/heads/master 9f76faa70 -> 10f085f9a
HAWQ-1198. Fix to filter out irrelevant fragments while accessing Hive. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/10f085f9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/10f085f9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/10f085f9 Branch: refs/heads/master Commit: 10f085f9aa2e4b00d775648cfbaf6059483b3dfc Parents: 9f76faa Author: Shubham Sharma <ssha...@pivotal.io> Authored: Thu Sep 14 16:24:27 2017 -0700 Committer: Oleksandr Diachenko <odiache...@pivotal.io> Committed: Thu Sep 14 16:24:27 2017 -0700 ---------------------------------------------------------------------- .../pxf/plugins/hive/HiveDataFragmenter.java | 16 ++- .../plugins/hive/HiveDataFragmenterTest.java | 111 +++++++++++++++++++ 2 files changed, 118 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/10f085f9/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java index 6e193c2..c24b552 100644 --- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java +++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java @@ -402,17 +402,11 @@ public class HiveDataFragmenter extends Fragmenter { // Let's look first at the filter BasicFilter bFilter = (BasicFilter) filter; - // In case this is not an "equality filter", we ignore this filter (no - // add to filter list) - if (!(bFilter.getOperation() == FilterParser.Operation.HDOP_EQ)) { - LOG.debug("Filter operator is not EQ, ignore this filter for hive : " - + filter); - return false; - } - // Extract column name and value int filterColumnIndex = bFilter.getColumn().index(); - String filterValue = bFilter.getConstant().constant().toString(); + // Avoids NullPointerException in case of operations like HDOP_IS_NULL, + // HDOP_IS_NOT_NULL where no constant value is passed as part of query + String filterValue = bFilter.getConstant()!= null ? bFilter.getConstant().constant().toString() : ""; ColumnDescriptor filterColumn = inputData.getColumn(filterColumnIndex); String filterColumnName = filterColumn.columnName(); @@ -453,6 +447,10 @@ public class HiveDataFragmenter extends Fragmenter { case HDOP_NE: filtersString.append(HIVE_API_NE); break; + default: + // Set filter string to blank in case of unimplemented operations + filtersString.setLength(0); + return false; } filtersString.append(HIVE_API_DQUOTE); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/10f085f9/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenterTest.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenterTest.java b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenterTest.java index f93f7ba..2c28500 100755 --- a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenterTest.java +++ b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenterTest.java @@ -20,7 +20,11 @@ package org.apache.hawq.pxf.plugins.hive; */ +import org.apache.hawq.pxf.api.FilterParser; import org.apache.hawq.pxf.api.utilities.InputData; +import org.apache.hawq.pxf.api.BasicFilter; +import org.apache.hawq.pxf.api.utilities.ColumnDescriptor; +import static org.apache.hawq.pxf.api.FilterParser.Operation.*; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; @@ -39,6 +43,10 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor; import org.powermock.modules.junit4.PowerMockRunner; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.*; + @RunWith(PowerMockRunner.class) @PrepareForTest({HiveDataFragmenter.class}) // Enables mocking 'new' calls @SuppressStaticInitializationFor({"org.apache.hadoop.mapred.JobConf", @@ -88,6 +96,109 @@ public class HiveDataFragmenterTest { } } + @Test + public void testBuildSingleFilter() throws Exception { + prepareConstruction(); + fragmenter = new HiveDataFragmenter(inputData); + ColumnDescriptor columnDescriptor = + new ColumnDescriptor("textColumn", 25, 3, "text", null,true); + String filterColumnName=columnDescriptor.columnName(); + int filterColumnIndex = columnDescriptor.columnIndex(); + HiveFilterBuilder builder = new HiveFilterBuilder(null); + when(inputData.getColumn(filterColumnIndex)).thenReturn(columnDescriptor); + + // Mock private field partitionkeyTypes + Field partitionkeyTypes = PowerMockito.field(HiveDataFragmenter.class, "partitionkeyTypes"); + Map<String, String> localpartitionkeyTypes = new HashMap<>(); + localpartitionkeyTypes.put(filterColumnName,"string"); + partitionkeyTypes.set(fragmenter,localpartitionkeyTypes); + + //Mock private field setPartitions + Field setPartitions = PowerMockito.field(HiveDataFragmenter.class, "setPartitions"); + Set<String> localSetPartitions = new TreeSet<String>( + String.CASE_INSENSITIVE_ORDER); + localSetPartitions.add(filterColumnName); + setPartitions.set(fragmenter,localSetPartitions); + + Map<FilterParser.Operation, String> filterStrings = new HashMap<>(); + /* + * Filter string representation in the respective order of their declaration + testColumn != 2016-01-03 + testColumn = 2016-01-03 + testColumn >= 2016-01-03 + testColumn <= 2016-01-03 + testColumn >= 2016-01-03 + testColumn < 2016-01-03 + testColumn like '2016-01-0%' + */ + filterStrings.put(HDOP_NE, "a3c25s10d2016-01-03o6"); + filterStrings.put(HDOP_EQ, "a3c25s10d2016-01-03o5"); + filterStrings.put(HDOP_GE, "a3c25s10d2016-01-03o4"); + filterStrings.put(HDOP_LE, "a3c25s10d2016-01-03o3"); + filterStrings.put(HDOP_GT, "a3c25s10d2016-01-03o2"); + filterStrings.put(HDOP_LT, "a3c25s10d2016-01-03o1"); + filterStrings.put(HDOP_LIKE, "a3c25s10d2016-01-0%o7"); + + for (FilterParser.Operation operation : filterStrings.keySet()){ + BasicFilter bFilter = (BasicFilter) builder.getFilterObject(filterStrings.get(operation)); + checkFilters(fragmenter,bFilter, operation); + } + } + + private void checkFilters(HiveDataFragmenter fragmenter, BasicFilter bFilter, FilterParser.Operation operation) + throws Exception{ + + String prefix=""; + StringBuilder localFilterString = new StringBuilder(); + String expectedResult; + + // Mock private method buildSingleFilter + Method method = PowerMockito.method(HiveDataFragmenter.class, "buildSingleFilter", + new Class[]{Object.class,StringBuilder.class,String.class}); + boolean result = (Boolean)method.invoke(fragmenter, new Object[]{bFilter,localFilterString,prefix}); + + switch (operation){ + case HDOP_NE: + expectedResult = "textColumn != \"2016-01-03\""; + assertTrue(result); + assertEquals(expectedResult,localFilterString.toString()); + break; + case HDOP_EQ: + expectedResult = "textColumn = \"2016-01-03\""; + assertTrue(result); + assertEquals(expectedResult,localFilterString.toString()); + break; + case HDOP_GE: + expectedResult = "textColumn >= \"2016-01-03\""; + assertTrue(result); + assertEquals(expectedResult,localFilterString.toString()); + break; + case HDOP_LE: + expectedResult = "textColumn <= \"2016-01-03\""; + assertTrue(result); + assertEquals(expectedResult,localFilterString.toString()); + break; + case HDOP_GT: + expectedResult = "textColumn > \"2016-01-03\""; + assertTrue(result); + assertEquals(expectedResult,localFilterString.toString()); + break; + case HDOP_LT: + expectedResult = "textColumn < \"2016-01-03\""; + assertTrue(result); + assertEquals(expectedResult,localFilterString.toString()); + break; + case HDOP_LIKE: + expectedResult = ""; + assertFalse(result); + assertEquals(expectedResult,localFilterString.toString()); + break; + default: + assertFalse(result); + break; + } + } + private void prepareConstruction() throws Exception { inputData = mock(InputData.class);