Repository: incubator-hawq Updated Branches: refs/heads/master c7a76523d -> d6dd24fd3
HAWQ-1431. Do not use StatsAccessor when column specified in SELECT clause. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/d6dd24fd Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/d6dd24fd Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/d6dd24fd Branch: refs/heads/master Commit: d6dd24fd31dd3cb6a7c9235f68d1c5dc32f11206 Parents: c7a7652 Author: Oleksandr Diachenko <odiache...@pivotal.io> Authored: Thu Apr 13 17:01:39 2017 -0700 Committer: Oleksandr Diachenko <odiache...@pivotal.io> Committed: Thu Apr 13 17:01:39 2017 -0700 ---------------------------------------------------------------------- .../hawq/pxf/api/utilities/InputData.java | 26 ++++++++++++++++++++ .../hawq/pxf/api/utilities/Utilities.java | 14 +++++------ .../hawq/pxf/api/utilities/UtilitiesTest.java | 7 ++++++ .../pxf/plugins/hive/HiveORCAccessorTest.java | 1 + .../pxf/service/utilities/ProtocolData.java | 1 + 5 files changed, 42 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d6dd24fd/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java index 959cda6..7bf57ed 100644 --- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java +++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java @@ -75,6 +75,16 @@ public class InputData { protected ColumnDescriptor recordkeyColumn; /** + * Number of attributes projected in query. + * + * Example: + * SELECT col1, col2, col3... : number of attributes projected - 3 + * SELECT col1, col2, col3... WHERE col4=a : number of attributes projected - 4 + * SELECT *... : number of attributes projected - 0 + */ + protected int numAttrsProjected; + + /** * Constructs an empty InputData */ public InputData() { @@ -369,4 +379,20 @@ public class InputData { this.fragmentIndex = fragmentIndex; } + /** + * Returns number of attributes projected in a query + * @return number of attributes projected + */ + public int getNumAttrsProjected() { + return numAttrsProjected; + } + + /** + * Sets number of attributes projected + * @param numAttrsProjected number of attrivutes projected + */ + public void setNumAttrsProjected(int numAttrsProjected) { + this.numAttrsProjected = numAttrsProjected; + } + } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d6dd24fd/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java index 6dd0044..916c2b5 100644 --- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java +++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java @@ -204,6 +204,9 @@ public class Utilities { public static boolean useAggBridge(InputData inputData) { boolean isStatsAccessor = false; try { + if (inputData == null || inputData.getAccessor() == null) { + throw new IllegalArgumentException("Missing accessor information"); + } isStatsAccessor = ArrayUtils.contains(Class.forName(inputData.getAccessor()).getInterfaces(), StatsAccessor.class); } catch (ClassNotFoundException e) { LOG.error("Unable to load accessor class: " + e.getMessage()); @@ -213,6 +216,7 @@ public class Utilities { return (inputData != null) && !inputData.hasFilter() && (inputData.getAggType() != null) && inputData.getAggType().isOptimizationSupported() + && inputData.getNumAttrsProjected() == 0 && isStatsAccessor; } @@ -224,14 +228,10 @@ public class Utilities { * @return true if this accessor should use statistic information */ public static boolean useStats(ReadAccessor accessor, InputData inputData) { - if (accessor instanceof StatsAccessor) { - /* Make sure filter is not present and aggregate operation supports optimization */ - if (inputData != null && !inputData.hasFilter() - && inputData.getAggType() != null - && inputData.getAggType().isOptimizationSupported()) { + if (accessor instanceof StatsAccessor && useAggBridge(inputData)) { return true; - } + } else { + return false; } - return false; } } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d6dd24fd/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/utilities/UtilitiesTest.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/utilities/UtilitiesTest.java b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/utilities/UtilitiesTest.java index 12ba3d1..01c09bf 100644 --- a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/utilities/UtilitiesTest.java +++ b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/utilities/UtilitiesTest.java @@ -189,6 +189,7 @@ public class UtilitiesTest { InputData metaData = mock(InputData.class); when(metaData.getAccessor()).thenReturn(StatsAccessorImpl.class.getName()); when(metaData.getAggType()).thenReturn(EnumAggregationType.COUNT); + when(metaData.getAccessor()).thenReturn("org.apache.hawq.pxf.api.utilities.UtilitiesTest$StatsAccessorImpl"); assertTrue(Utilities.useAggBridge(metaData)); when(metaData.getAccessor()).thenReturn(UtilitiesTest.class.getName()); @@ -207,6 +208,7 @@ public class UtilitiesTest { InputData metaData = mock(InputData.class); ReadAccessor accessor = new StatsAccessorImpl(); when(metaData.getAggType()).thenReturn(EnumAggregationType.COUNT); + when(metaData.getAccessor()).thenReturn("org.apache.hawq.pxf.api.utilities.UtilitiesTest$StatsAccessorImpl"); assertTrue(Utilities.useStats(accessor, metaData)); ReadAccessor nonStatusAccessor = new NonStatsAccessorImpl(); assertFalse(Utilities.useStats(nonStatusAccessor, metaData)); @@ -214,5 +216,10 @@ public class UtilitiesTest { //Do not use stats when input data has filter when(metaData.hasFilter()).thenReturn(true); assertFalse(Utilities.useStats(accessor, metaData)); + + //Do not use stats when more than one column is projected + when(metaData.hasFilter()).thenReturn(false); + when(metaData.getNumAttrsProjected()).thenReturn(1); + assertFalse(Utilities.useStats(accessor, metaData)); } } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d6dd24fd/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessorTest.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessorTest.java b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessorTest.java index daee331..b1ee3f3 100644 --- a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessorTest.java +++ b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessorTest.java @@ -82,6 +82,7 @@ public class HiveORCAccessorTest { PowerMockito.whenNew(OrcInputFormat.class).withNoArguments().thenReturn(orcInputFormat); RecordReader recordReader = mock(RecordReader.class); PowerMockito.when(orcInputFormat.getRecordReader(any(InputSplit.class), any(JobConf.class), any(Reporter.class))).thenReturn(recordReader); + PowerMockito.when(inputData.getAccessor()).thenReturn(HiveORCAccessor.class.getName()); accessor = new HiveORCAccessor(inputData); } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d6dd24fd/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java index 0cb6d47..8456165 100644 --- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java +++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java @@ -383,6 +383,7 @@ public class ProtocolData extends InputData { List<Integer> columnProjList = new ArrayList<Integer>(); if(columnProjStr != null) { int columnProj = Integer.parseInt(columnProjStr); + numAttrsProjected = columnProj; if(columnProj > 0) { String columnProjIndexStr = getProperty("ATTRS-PROJ-IDX"); String columnProjIdx[] = columnProjIndexStr.split(",");