PHOENIX-4645 PhoenixStorageHandler doesn't handle correctly data/timestamp in push down predicate when engine is tez.(Sergey Soldatov)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/8f1cef82 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/8f1cef82 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/8f1cef82 Branch: refs/heads/system-catalog Commit: 8f1cef824b086c7c697688767e0460c18fa554d6 Parents: 78636a3 Author: Rajeshbabu Chintaguntla <rajeshb...@apache.org> Authored: Tue Apr 24 11:24:43 2018 +0530 Committer: Rajeshbabu Chintaguntla <rajeshb...@apache.org> Committed: Tue Apr 24 11:24:43 2018 +0530 ---------------------------------------------------------------------- .../apache/phoenix/hive/HivePhoenixStoreIT.java | 27 ++++++++++++++++++++ .../PhoenixStorageHandlerConstants.java | 8 +++--- .../phoenix/hive/query/PhoenixQueryBuilder.java | 4 +-- 3 files changed, 33 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/8f1cef82/phoenix-hive/src/it/java/org/apache/phoenix/hive/HivePhoenixStoreIT.java ---------------------------------------------------------------------- diff --git a/phoenix-hive/src/it/java/org/apache/phoenix/hive/HivePhoenixStoreIT.java b/phoenix-hive/src/it/java/org/apache/phoenix/hive/HivePhoenixStoreIT.java index 1828818..66f99ad 100644 --- a/phoenix-hive/src/it/java/org/apache/phoenix/hive/HivePhoenixStoreIT.java +++ b/phoenix-hive/src/it/java/org/apache/phoenix/hive/HivePhoenixStoreIT.java @@ -309,6 +309,33 @@ public class HivePhoenixStoreIT extends BaseHivePhoenixStoreIT { assert (rs.getString(2).equalsIgnoreCase("part1")); assert (rs.getDouble(3) == 200); } + } + + @Test + public void testTimestampPredicate() throws Exception { + String testName = "testTimeStampPredicate"; + hbaseTestUtil.getTestFileSystem().createNewFile(new Path(hiveLogDir, testName + ".out")); + createFile("10\t2013-01-02 01:01:01.123456\n", new Path(hiveOutputDir, testName + ".out").toString()); + createFile(StringUtil.EMPTY_STRING, new Path(hiveLogDir, testName + ".out").toString()); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE TABLE timeStampTable(ID int,ts TIMESTAMP)" + HiveTestUtil.CRLF + + " STORED BY \"org.apache.phoenix.hive.PhoenixStorageHandler\"" + HiveTestUtil + .CRLF + + " TBLPROPERTIES(" + HiveTestUtil.CRLF + + " 'phoenix.hbase.table.name'='TIMESTAMPTABLE'," + HiveTestUtil.CRLF + + " 'phoenix.zookeeper.znode.parent'='/hbase'," + HiveTestUtil.CRLF + + " 'phoenix.zookeeper.quorum'='localhost'," + HiveTestUtil.CRLF + + " 'phoenix.zookeeper.client.port'='" + + hbaseTestUtil.getZkCluster().getClientPort() + "'," + HiveTestUtil.CRLF + + " 'phoenix.column.mapping' = 'id:ID, ts:TS'," + HiveTestUtil.CRLF + + " 'phoenix.rowkeys'='id');" + HiveTestUtil.CRLF); + sb.append("INSERT INTO TABLE timeStampTable VALUES (10, \"2013-01-02 01:01:01.123456\");" + HiveTestUtil.CRLF); + sb.append("SELECT * from timeStampTable WHERE ts between '2013-01-02 01:01:01.123455' and " + + " '2013-01-02 12:01:02.123457789' AND id = 10;" + HiveTestUtil.CRLF); + + String fullPath = new Path(hbaseTestUtil.getDataTestDir(), testName).toString(); + createFile(sb.toString(), fullPath); + runTest(testName, fullPath); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/8f1cef82/phoenix-hive/src/main/java/org/apache/phoenix/hive/constants/PhoenixStorageHandlerConstants.java ---------------------------------------------------------------------- diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/constants/PhoenixStorageHandlerConstants.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/constants/PhoenixStorageHandlerConstants.java index 07c374e..e3c7d84 100644 --- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/constants/PhoenixStorageHandlerConstants.java +++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/constants/PhoenixStorageHandlerConstants.java @@ -90,13 +90,13 @@ public class PhoenixStorageHandlerConstants { public static final String PATERN_MARKER = "$targetPattern$"; public static final String DATE_PATTERN = "'?\\d{4}-\\d{2}-\\d{2}'?"; public static final String TIMESTAMP_PATTERN = "'?\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\" + - ".?\\d{0,3}'?"; - public static final String COMMON_OPERATOR_PATTERN = "(\\(?" + COLUMNE_MARKER + "\\)?\\s*" + + ".?\\d{0,9}'?"; + public static final String COMMON_OPERATOR_PATTERN = "(\\(?\"?" + COLUMNE_MARKER + "\"?\\)?\\s*" + "(=|>|<|<=|>=)\\s*(" + PATERN_MARKER + "))"; - public static final String BETWEEN_OPERATOR_PATTERN = "(\\(?" + COLUMNE_MARKER + "\\)?\\s*(" + + public static final String BETWEEN_OPERATOR_PATTERN = "(\\(?\"?" + COLUMNE_MARKER + "\"?\\)?\\s*(" + "(?i)not)?\\s*(?i)between\\s*(" + PATERN_MARKER + ")\\s*(?i)and\\s*(" + PATERN_MARKER + "))"; - public static final String IN_OPERATOR_PATTERN = "(\\(?" + COLUMNE_MARKER + "\\)?\\s*((?i)" + + public static final String IN_OPERATOR_PATTERN = "(\\(?\"?" + COLUMNE_MARKER + "\"?\\)?\\s*((?i)" + "not)?\\s*(?i)in\\s*\\((" + PATERN_MARKER + ",?\\s*)+\\))"; public static final String FUNCTION_VALUE_MARKER = "$value$"; http://git-wip-us.apache.org/repos/asf/phoenix/blob/8f1cef82/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java ---------------------------------------------------------------------- diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java index eb336dc..ab409ad 100644 --- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java +++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java @@ -255,10 +255,10 @@ public class PhoenixQueryBuilder { if (PhoenixStorageHandlerConstants.DATE_TYPE.equals( columnTypeMap.get(columnName).getTypeName())) { - whereClause = applyDateFunctionUsingRegex(whereClause, columnName); + whereClause = applyDateFunctionUsingRegex(whereClause, column); } else if (PhoenixStorageHandlerConstants.TIMESTAMP_TYPE.equals( columnTypeMap.get(columnName).getTypeName())) { - whereClause = applyTimestampFunctionUsingRegex(whereClause, columnName); + whereClause = applyTimestampFunctionUsingRegex(whereClause, column); } } }
