PHOENIX-1826 Implement TrackOrderPreservingExpressionCompiler as Expression visitor instead of ParseNode visitor
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/b4f0f167 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/b4f0f167 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/b4f0f167 Branch: refs/heads/4.x-HBase-0.98 Commit: b4f0f16779378e12f3c819307c6fe2e609a856b7 Parents: f8c80e5 Author: James Taylor <jtay...@salesforce.com> Authored: Mon Apr 13 16:27:20 2015 -0700 Committer: Cody Marcel <codymar...@apache.org> Committed: Tue Apr 14 09:40:14 2015 -0700 ---------------------------------------------------------------------- .../apache/phoenix/end2end/DerivedTableIT.java | 4 +- .../org/apache/phoenix/end2end/HashJoinIT.java | 7 - .../org/apache/phoenix/end2end/OrderByIT.java | 13 +- .../org/apache/phoenix/end2end/SubqueryIT.java | 8 +- .../end2end/SubqueryUsingSortMergeJoinIT.java | 20 +- .../phoenix/end2end/VariableLengthPKIT.java | 2 +- .../index/GlobalIndexOptimizationIT.java | 11 +- .../apache/phoenix/compile/GroupByCompiler.java | 56 ++-- .../apache/phoenix/compile/OrderByCompiler.java | 46 ++-- .../phoenix/compile/OrderPreservingTracker.java | 259 +++++++++++++++++++ .../TrackOrderPreservingExpressionCompiler.java | 249 ------------------ .../phoenix/compile/QueryCompilerTest.java | 108 +++++++- .../phoenix/compile/QueryOptimizerTest.java | 7 +- 13 files changed, 441 insertions(+), 349 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/b4f0f167/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java index 187025a..e8ed99d 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java @@ -308,8 +308,8 @@ public class DerivedTableIT extends BaseClientManagedTimeIT { rs = conn.createStatement().executeQuery("EXPLAIN " + query); assertEquals(plans[0], QueryUtil.getExplainPlan(rs)); - // distinct b (groupby b, a) groupby a - query = "SELECT DISTINCT COLLECTDISTINCT(t.b) FROM (SELECT b_string b, a_string a FROM aTable GROUP BY b_string, a_string) AS t GROUP BY t.a"; + // distinct b (groupby a, b) groupby a + query = "SELECT DISTINCT COLLECTDISTINCT(t.b) FROM (SELECT b_string b, a_string a FROM aTable GROUP BY a_string, b_string) AS t GROUP BY t.a"; statement = conn.prepareStatement(query); rs = statement.executeQuery(); assertTrue (rs.next()); http://git-wip-us.apache.org/repos/asf/phoenix/blob/b4f0f167/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java index 1a2a1d0..a03204a 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java @@ -118,7 +118,6 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT { "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_DISPLAY_NAME + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [I.NAME]\n" + "CLIENT MERGE SORT\n" + - "CLIENT SORTED BY [I.NAME]\n" + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_DISPLAY_NAME, /* @@ -156,7 +155,6 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT { "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_DISPLAY_NAME + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [I.NAME]\n" + "CLIENT MERGE SORT\n" + - "CLIENT SORTED BY [I.NAME]\n" + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_DISPLAY_NAME, /* @@ -307,7 +305,6 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT { "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_DISPLAY_NAME + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [I.NAME]\n" + "CLIENT MERGE SORT\n" + - "CLIENT SORTED BY [I.NAME]\n" + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_DISPLAY_NAME, /* @@ -495,7 +492,6 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT { "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_DISPLAY_NAME + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"I.0:NAME\"]\n" + "CLIENT MERGE SORT\n" + - "CLIENT SORTED BY [\"I.0:NAME\"]\n" + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" + " SERVER FILTER BY FIRST KEY ONLY", @@ -687,7 +683,6 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT { "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_DISPLAY_NAME + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [I.NAME]\n" + "CLIENT MERGE SORT\n" + - "CLIENT SORTED BY [I.NAME]\n" + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" + " SERVER FILTER BY FIRST KEY ONLY", @@ -876,7 +871,6 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT { "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_DISPLAY_NAME + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"I.0:NAME\"]\n" + "CLIENT MERGE SORT\n" + - "CLIENT SORTED BY [\"I.0:NAME\"]\n" + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + MetaDataUtil.LOCAL_INDEX_TABLE_PREFIX + "" + JOIN_ITEM_TABLE_DISPLAY_NAME +" [-32768]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + @@ -1085,7 +1079,6 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT { "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_DISPLAY_NAME + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [I.NAME]\n" + "CLIENT MERGE SORT\n" + - "CLIENT SORTED BY [I.NAME]\n" + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + MetaDataUtil.LOCAL_INDEX_TABLE_PREFIX +""+JOIN_ITEM_TABLE_DISPLAY_NAME+" [-32768]\n"+ " SERVER FILTER BY FIRST KEY ONLY\n" + http://git-wip-us.apache.org/repos/asf/phoenix/blob/b4f0f167/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByIT.java index 74eb7fe..9fc3003 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByIT.java @@ -30,7 +30,6 @@ import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; import java.sql.Connection; import java.sql.Date; @@ -499,9 +498,15 @@ public class OrderByIT extends BaseClientManagedTimeIT { stmt.execute(); conn.commit(); - String query = "SELECT col1+col2, col4, TRUNC(col3, 'HOUR') FROM e_table ORDER BY 1, 2"; - conn.createStatement().executeQuery(query); - fail(); + String query = "SELECT col1+col2, col4, a_string FROM e_table ORDER BY 1, 2"; + ResultSet rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("a", rs.getString(3)); + assertTrue(rs.next()); + assertEquals("c", rs.getString(3)); + assertTrue(rs.next()); + assertEquals("b", rs.getString(3)); + assertFalse(rs.next()); } catch (SQLException e) { } finally { conn.close(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/b4f0f167/phoenix-core/src/it/java/org/apache/phoenix/end2end/SubqueryIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SubqueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SubqueryIT.java index f655e0a..13354da 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SubqueryIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SubqueryIT.java @@ -200,7 +200,7 @@ public class SubqueryIT extends BaseHBaseManagedTimeIT { " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + - " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.0:NAME\", \".+.:item_id\"\\]\n" + + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" + " CLIENT MERGE SORT\n" + " PARALLEL ANTI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_DISPLAY_NAME + "\n" + @@ -209,7 +209,7 @@ public class SubqueryIT extends BaseHBaseManagedTimeIT { " PARALLEL LEFT-JOIN TABLE 1\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + - " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.0:NAME\", \".+.:item_id\"\\]\n" + + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" + " CLIENT MERGE SORT\n" + " PARALLEL SEMI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_DISPLAY_NAME + "\n" + @@ -274,7 +274,7 @@ public class SubqueryIT extends BaseHBaseManagedTimeIT { " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + MetaDataUtil.LOCAL_INDEX_TABLE_PREFIX + JOIN_ITEM_TABLE_DISPLAY_NAME + " \\[-32768\\]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + - " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.0:NAME\", \".+.:item_id\"\\]\n" + + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" + " CLIENT MERGE SORT\n" + " PARALLEL ANTI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_DISPLAY_NAME + "\n" + @@ -283,7 +283,7 @@ public class SubqueryIT extends BaseHBaseManagedTimeIT { " PARALLEL LEFT-JOIN TABLE 1\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + MetaDataUtil.LOCAL_INDEX_TABLE_PREFIX + JOIN_ITEM_TABLE_DISPLAY_NAME + " \\[-32768\\]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + - " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.0:NAME\", \".+.:item_id\"\\]\n" + + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" + " CLIENT MERGE SORT\n" + " PARALLEL SEMI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_DISPLAY_NAME + "\n" + http://git-wip-us.apache.org/repos/asf/phoenix/blob/b4f0f167/phoenix-core/src/it/java/org/apache/phoenix/end2end/SubqueryUsingSortMergeJoinIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SubqueryUsingSortMergeJoinIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SubqueryUsingSortMergeJoinIT.java index 59f75e5..cb9f4b1 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SubqueryUsingSortMergeJoinIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SubqueryUsingSortMergeJoinIT.java @@ -121,7 +121,6 @@ public class SubqueryUsingSortMergeJoinIT extends BaseHBaseManagedTimeIT { " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ORDER_TABLE_DISPLAY_NAME + " ['000000000000001'] - [*]\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]\n" + " CLIENT MERGE SORT\n" + - " CLIENT SORTED BY [\"item_id\"]\n" + "CLIENT SORTED BY [I.NAME]", "SORT-MERGE-JOIN \\(LEFT\\) TABLES\n" + @@ -132,7 +131,6 @@ public class SubqueryUsingSortMergeJoinIT extends BaseHBaseManagedTimeIT { " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_DISPLAY_NAME + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\".+.item_id\", .+.NAME\\]\n" + " CLIENT MERGE SORT\n" + - " CLIENT SORTED BY \\[\".+.item_id\", .+.NAME\\]\n" + " PARALLEL ANTI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_DISPLAY_NAME + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"]\\\n" + @@ -142,7 +140,6 @@ public class SubqueryUsingSortMergeJoinIT extends BaseHBaseManagedTimeIT { " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_DISPLAY_NAME + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\".+.item_id\", .+.NAME\\]\n" + " CLIENT MERGE SORT\n" + - " CLIENT SORTED BY \\[\".+.item_id\", .+.NAME\\]\n" + " SKIP-SCAN-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_DISPLAY_NAME + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" + @@ -156,7 +153,6 @@ public class SubqueryUsingSortMergeJoinIT extends BaseHBaseManagedTimeIT { " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_DISPLAY_NAME + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"O.customer_id\"\\]\n" + " CLIENT MERGE SORT\n" + - " CLIENT SORTED BY \\[\"O.customer_id\"\\]\n" + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_DISPLAY_NAME + "\n" + " PARALLEL LEFT-JOIN TABLE 1\\(DELAYED EVALUATION\\)\n" + @@ -187,7 +183,6 @@ public class SubqueryUsingSortMergeJoinIT extends BaseHBaseManagedTimeIT { " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ORDER_TABLE_DISPLAY_NAME + " ['000000000000001'] - [*]\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]\n" + " CLIENT MERGE SORT\n" + - " CLIENT SORTED BY [\"item_id\"]\n" + "CLIENT SORTED BY [\"I.0:NAME\"]", "SORT-MERGE-JOIN \\(LEFT\\) TABLES\n" + @@ -197,9 +192,8 @@ public class SubqueryUsingSortMergeJoinIT extends BaseHBaseManagedTimeIT { " AND\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + - " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.0:NAME\", \".+.:item_id\"\\]\n" + + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" + " CLIENT MERGE SORT\n" + - " CLIENT SORTED BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" + " PARALLEL ANTI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_DISPLAY_NAME + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" + @@ -208,9 +202,8 @@ public class SubqueryUsingSortMergeJoinIT extends BaseHBaseManagedTimeIT { "AND\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + - " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.0:NAME\", \".+.:item_id\"\\]\n" + + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" + " CLIENT MERGE SORT\n" + - " CLIENT SORTED BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" + " PARALLEL SEMI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_DISPLAY_NAME + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" + @@ -227,7 +220,6 @@ public class SubqueryUsingSortMergeJoinIT extends BaseHBaseManagedTimeIT { " SERVER FILTER BY FIRST KEY ONLY\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"O.customer_id\"\\]\n" + " CLIENT MERGE SORT\n" + - " CLIENT SORTED BY \\[\"O.customer_id\"\\]\n" + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_DISPLAY_NAME + "\n" + " PARALLEL LEFT-JOIN TABLE 1\\(DELAYED EVALUATION\\)\n" + @@ -257,7 +249,6 @@ public class SubqueryUsingSortMergeJoinIT extends BaseHBaseManagedTimeIT { " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ORDER_TABLE_DISPLAY_NAME + " ['000000000000001'] - [*]\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]\n" + " CLIENT MERGE SORT\n" + - " CLIENT SORTED BY [\"item_id\"]\n" + "CLIENT SORTED BY [\"I.0:NAME\"]", "SORT-MERGE-JOIN \\(LEFT\\) TABLES\n" + @@ -267,9 +258,8 @@ public class SubqueryUsingSortMergeJoinIT extends BaseHBaseManagedTimeIT { " AND\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + MetaDataUtil.LOCAL_INDEX_TABLE_PREFIX + JOIN_ITEM_TABLE_DISPLAY_NAME + " \\[-32768\\]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + - " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.0:NAME\", \".+.:item_id\"\\]\n" + + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" + " CLIENT MERGE SORT\n" + - " CLIENT SORTED BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" + " PARALLEL ANTI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_DISPLAY_NAME + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" + @@ -278,9 +268,8 @@ public class SubqueryUsingSortMergeJoinIT extends BaseHBaseManagedTimeIT { "AND\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + MetaDataUtil.LOCAL_INDEX_TABLE_PREFIX + JOIN_ITEM_TABLE_DISPLAY_NAME + " \\[-32768\\]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + - " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.0:NAME\", \".+.:item_id\"\\]\n" + + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" + " CLIENT MERGE SORT\n" + - " CLIENT SORTED BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" + " PARALLEL SEMI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_DISPLAY_NAME + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" + @@ -298,7 +287,6 @@ public class SubqueryUsingSortMergeJoinIT extends BaseHBaseManagedTimeIT { " SERVER FILTER BY FIRST KEY ONLY\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"O.customer_id\"\\]\n" + " CLIENT MERGE SORT\n" + - " CLIENT SORTED BY \\[\"O.customer_id\"\\]\n" + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_DISPLAY_NAME + "\n" + " PARALLEL LEFT-JOIN TABLE 1\\(DELAYED EVALUATION\\)\n" + http://git-wip-us.apache.org/repos/asf/phoenix/blob/b4f0f167/phoenix-core/src/it/java/org/apache/phoenix/end2end/VariableLengthPKIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/VariableLengthPKIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/VariableLengthPKIT.java index b7bc7cc..1e48f8c 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/VariableLengthPKIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/VariableLengthPKIT.java @@ -1090,7 +1090,7 @@ public class VariableLengthPKIT extends BaseClientManagedTimeIT { @Test public void testMultiFixedLengthNull() throws Exception { long ts = nextTimestamp(); - String query = "SELECT B_INTEGER,C_INTEGER,COUNT(1) FROM BTABLE GROUP BY C_INTEGER,B_INTEGER"; + String query = "SELECT B_INTEGER,C_INTEGER,COUNT(1) FROM BTABLE GROUP BY B_INTEGER,C_INTEGER"; String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + (ts + 5); // Run query at timestamp 5 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); Connection conn = DriverManager.getConnection(url, props); http://git-wip-us.apache.org/repos/asf/phoenix/blob/b4f0f167/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java index 07d87b7..b97176f 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java @@ -198,7 +198,7 @@ public class GlobalIndexOptimizationIT extends BaseHBaseManagedTimeIT { expected = "CLIENT PARALLEL \\d-WAY FULL SCAN OVER " + TestUtil.DEFAULT_DATA_TABLE_NAME + "\n" + - " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[T.T_ID, T.V1, T.K3\\]\n" + + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[T.V1, T.T_ID, T.K3\\]\n" + "CLIENT MERGE SORT\n" + " SKIP-SCAN-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + TestUtil.DEFAULT_INDEX_TABLE_NAME + " \\[\\*\\] - \\['z'\\]\n" + @@ -209,10 +209,6 @@ public class GlobalIndexOptimizationIT extends BaseHBaseManagedTimeIT { rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); - assertEquals("b", rs.getString("t_id"));; - assertEquals(4, rs.getInt("k3")); - assertEquals("z", rs.getString("V1")); - assertTrue(rs.next()); assertEquals("f", rs.getString("t_id")); assertEquals(3, rs.getInt("k3")); assertEquals("a", rs.getString("V1")); @@ -224,6 +220,10 @@ public class GlobalIndexOptimizationIT extends BaseHBaseManagedTimeIT { assertEquals("q", rs.getString("t_id")); assertEquals(1, rs.getInt("k3")); assertEquals("c", rs.getString("V1")); + assertTrue(rs.next()); + assertEquals("b", rs.getString("t_id"));; + assertEquals(4, rs.getInt("k3")); + assertEquals("z", rs.getString("V1")); assertFalse(rs.next()); query = "SELECT /*+ INDEX(" + TestUtil.DEFAULT_DATA_TABLE_NAME + " " + TestUtil.DEFAULT_INDEX_TABLE_NAME + ")*/ v1,sum(k3) from " + TestUtil.DEFAULT_DATA_TABLE_FULL_NAME + " where v1 <='z' group by v1 order by v1"; @@ -233,7 +233,6 @@ public class GlobalIndexOptimizationIT extends BaseHBaseManagedTimeIT { "CLIENT PARALLEL \\d-WAY FULL SCAN OVER T\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[T.V1\\]\n" + "CLIENT MERGE SORT\n" + - "CLIENT SORTED BY \\[T.V1\\]\n" + " SKIP-SCAN-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER I \\[\\*\\] - \\['z'\\]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + http://git-wip-us.apache.org/repos/asf/phoenix/blob/b4f0f167/phoenix-core/src/main/java/org/apache/phoenix/compile/GroupByCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/GroupByCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/GroupByCompiler.java index 4f1ba5b..7d9df02 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/GroupByCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/GroupByCompiler.java @@ -23,9 +23,9 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; +import org.apache.hadoop.hbase.util.Pair; import org.apache.http.annotation.Immutable; -import org.apache.phoenix.compile.TrackOrderPreservingExpressionCompiler.Entry; -import org.apache.phoenix.compile.TrackOrderPreservingExpressionCompiler.Ordering; +import org.apache.phoenix.compile.OrderPreservingTracker.Ordering; import org.apache.phoenix.coprocessor.BaseScannerRegionObserver; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.exception.SQLExceptionInfo; @@ -158,29 +158,30 @@ public class GroupByCompiler { } // Accumulate expressions in GROUP BY - TrackOrderPreservingExpressionCompiler groupByVisitor = - new TrackOrderPreservingExpressionCompiler(context, - GroupBy.EMPTY_GROUP_BY, groupByNodes.size(), - Ordering.UNORDERED, tupleProjector); - for (ParseNode node : groupByNodes) { - Expression expression = node.accept(groupByVisitor); - if (groupByVisitor.isAggregate()) { - throw new SQLExceptionInfo.Builder(SQLExceptionCode.AGGREGATE_IN_GROUP_BY) - .setMessage(expression.toString()).build().buildException(); - } + ExpressionCompiler compiler = + new ExpressionCompiler(context, GroupBy.EMPTY_GROUP_BY); + List<Pair<Integer,Expression>> groupBys = Lists.newArrayListWithExpectedSize(groupByNodes.size()); + OrderPreservingTracker tracker = new OrderPreservingTracker(context, GroupBy.EMPTY_GROUP_BY, Ordering.UNORDERED, groupByNodes.size(), tupleProjector); + for (int i = 0; i < groupByNodes.size(); i++) { + ParseNode node = groupByNodes.get(i); + Expression expression = node.accept(compiler); if (!expression.isStateless()) { - groupByVisitor.addEntry(expression); + if (compiler.isAggregate()) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.AGGREGATE_IN_GROUP_BY) + .setMessage(expression.toString()).build().buildException(); + } + tracker.track(expression); + groupBys.add(new Pair<Integer,Expression>(i,expression)); } - groupByVisitor.reset(); + compiler.reset(); } - List<Entry> groupByEntries = groupByVisitor.getEntries(); - if (groupByEntries.isEmpty()) { + if (groupBys.isEmpty()) { return GroupBy.EMPTY_GROUP_BY; } - boolean isRowKeyOrderedGrouping = isInRowKeyOrder && groupByVisitor.isOrderPreserving(); - List<Expression> expressions = Lists.newArrayListWithCapacity(groupByEntries.size()); + boolean isRowKeyOrderedGrouping = isInRowKeyOrder && tracker.isOrderPreserving(); + List<Expression> expressions = Lists.newArrayListWithExpectedSize(groupBys.size()); List<Expression> keyExpressions = expressions; String groupExprAttribName; // This is true if the GROUP BY is composed of only PK columns. We further check here that @@ -188,8 +189,8 @@ public class GroupByCompiler { // column and use each subsequent one in PK order). if (isRowKeyOrderedGrouping) { groupExprAttribName = BaseScannerRegionObserver.KEY_ORDERED_GROUP_BY_EXPRESSIONS; - for (Entry groupByEntry : groupByEntries) { - expressions.add(groupByEntry.getExpression()); + for (Pair<Integer,Expression> groupBy : groupBys) { + expressions.add(groupBy.getSecond()); } } else { /* @@ -211,11 +212,11 @@ public class GroupByCompiler { * Within each bucket, order based on the column position in the schema. Putting the fixed width values * in the beginning optimizes access to subsequent values. */ - Collections.sort(groupByEntries, new Comparator<Entry>() { + Collections.sort(groupBys, new Comparator<Pair<Integer,Expression>>() { @Override - public int compare(Entry o1, Entry o2) { - Expression e1 = o1.getExpression(); - Expression e2 = o2.getExpression(); + public int compare(Pair<Integer,Expression> gb1, Pair<Integer,Expression> gb2) { + Expression e1 = gb1.getSecond(); + Expression e2 = gb2.getSecond(); boolean isFixed1 = e1.getDataType().isFixedWidth(); boolean isFixed2 = e2.getDataType().isFixedWidth(); boolean isFixedNullable1 = e1.isNullable() &&isFixed1; @@ -224,7 +225,8 @@ public class GroupByCompiler { if (isFixed1 == isFixed2) { // Not strictly necessary, but forces the order to match the schema // column order (with PK columns before value columns). - return o1.getColumnPosition() - o2.getColumnPosition(); + //return o1.getColumnPosition() - o2.getColumnPosition(); + return gb1.getFirst() - gb2.getFirst(); } else if (isFixed1) { return -1; } else { @@ -237,8 +239,8 @@ public class GroupByCompiler { } } }); - for (Entry groupByEntry : groupByEntries) { - expressions.add(groupByEntry.getExpression()); + for (Pair<Integer,Expression> groupBy : groupBys) { + expressions.add(groupBy.getSecond()); } for (int i = expressions.size()-2; i >= 0; i--) { Expression expression = expressions.get(i); http://git-wip-us.apache.org/repos/asf/phoenix/blob/b4f0f167/phoenix-core/src/main/java/org/apache/phoenix/compile/OrderByCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/OrderByCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/OrderByCompiler.java index d8e86ad..f0406d4 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/OrderByCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/OrderByCompiler.java @@ -24,17 +24,15 @@ import java.util.LinkedHashSet; import java.util.List; import org.apache.phoenix.compile.GroupByCompiler.GroupBy; -import org.apache.phoenix.compile.TrackOrderPreservingExpressionCompiler.Ordering; +import org.apache.phoenix.compile.OrderPreservingTracker.Ordering; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.exception.SQLExceptionInfo; import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.OrderByExpression; -import org.apache.phoenix.parse.ColumnParseNode; import org.apache.phoenix.parse.LiteralParseNode; import org.apache.phoenix.parse.OrderByNode; import org.apache.phoenix.parse.ParseNode; import org.apache.phoenix.parse.SelectStatement; -import org.apache.phoenix.parse.TableName; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.PTableType; @@ -90,13 +88,12 @@ public class OrderByCompiler { if (orderByNodes.isEmpty()) { return OrderBy.EMPTY_ORDER_BY; } + ExpressionCompiler compiler = new ExpressionCompiler(context, groupBy); // accumulate columns in ORDER BY - TrackOrderPreservingExpressionCompiler visitor = - new TrackOrderPreservingExpressionCompiler(context, groupBy, - orderByNodes.size(), Ordering.ORDERED, null); + OrderPreservingTracker tracker = + new OrderPreservingTracker(context, groupBy, Ordering.ORDERED, orderByNodes.size()); LinkedHashSet<OrderByExpression> orderByExpressions = Sets.newLinkedHashSetWithExpectedSize(orderByNodes.size()); for (OrderByNode node : orderByNodes) { - boolean isAscending = node.isAscending(); ParseNode parseNode = node.getNode(); Expression expression = null; if (parseNode instanceof LiteralParseNode && ((LiteralParseNode)parseNode).getType() == PInteger.INSTANCE){ @@ -104,24 +101,13 @@ public class OrderByCompiler { int size = projector.getColumnProjectors().size(); if (index > size || index <= 0 ) { throw new SQLExceptionInfo.Builder(SQLExceptionCode.PARAM_INDEX_OUT_OF_BOUND) - .setMessage("").build().buildException(); + .build().buildException(); } - ColumnProjector colProj = projector.getColumnProjector(index-1); - TableName tableName = null; - if (statement.getSelects().size() > 0 ) - tableName = TableName.create(context.getCurrentTable().getTable().getName().toString(), null); - else { - tableName = TableName.create(context.getResolver().getTables().get(0).getTable().getSchemaName().toString(), - context.getResolver().getTables().get(0).getTable().getTableName().toString()); - } - ColumnParseNode colParseNode = new ColumnParseNode(tableName, colProj.getName(), null); - expression = colParseNode.accept(visitor); + expression = projector.getColumnProjector(index-1).getExpression(); } else { - expression = node.getNode().accept(visitor); - } - if (!expression.isStateless() && visitor.addEntry(expression, isAscending ? SortOrder.ASC : SortOrder.DESC)) { + expression = node.getNode().accept(compiler); // Detect mix of aggregate and non aggregates (i.e. ORDER BY txns, SUM(txns) - if (!visitor.isAggregate()) { + if (!expression.isStateless() && !compiler.isAggregate()) { if (statement.isAggregate() || statement.isDistinct()) { // Detect ORDER BY not in SELECT DISTINCT: SELECT DISTINCT count(*) FROM t ORDER BY x if (statement.isDistinct()) { @@ -131,21 +117,29 @@ public class OrderByCompiler { ExpressionCompiler.throwNonAggExpressionInAggException(expression.toString()); } } + } + if (!expression.isStateless()) { + boolean isAscending = node.isAscending(); + boolean isNullsLast = node.isNullsLast(); + tracker.track(expression, isAscending ? SortOrder.ASC : SortOrder.DESC, isNullsLast); + // FIXME: this isn't correct. If we have a schema where column A is DESC, + // An ORDER BY A should still be ASC. if (expression.getSortOrder() == SortOrder.DESC) { isAscending = !isAscending; + isNullsLast = !isNullsLast; } - OrderByExpression orderByExpression = new OrderByExpression(expression, node.isNullsLast(), isAscending); + OrderByExpression orderByExpression = new OrderByExpression(expression, isNullsLast, isAscending); orderByExpressions.add(orderByExpression); } - visitor.reset(); + compiler.reset(); } if (orderByExpressions.isEmpty()) { return OrderBy.EMPTY_ORDER_BY; } // If we're ordering by the order returned by the scan, we don't need an order by - if (isInRowKeyOrder && visitor.isOrderPreserving()) { - if (visitor.isReverse()) { + if (isInRowKeyOrder && tracker.isOrderPreserving()) { + if (tracker.isReverse()) { // Don't use reverse scan if we're using a skip scan, as our skip scan doesn't support this yet. // REV_ROW_KEY_ORDER_BY scan would not take effect for a projected table, so don't return it for such table types. if (context.getConnection().getQueryServices().getProps().getBoolean(QueryServices.USE_REVERSE_SCAN_ATTRIB, QueryServicesOptions.DEFAULT_USE_REVERSE_SCAN) http://git-wip-us.apache.org/repos/asf/phoenix/blob/b4f0f167/phoenix-core/src/main/java/org/apache/phoenix/compile/OrderPreservingTracker.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/OrderPreservingTracker.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/OrderPreservingTracker.java new file mode 100644 index 0000000..1c31606 --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/OrderPreservingTracker.java @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by + * applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.apache.phoenix.compile; + +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; + +import org.apache.phoenix.compile.GroupByCompiler.GroupBy; +import org.apache.phoenix.execute.TupleProjector; +import org.apache.phoenix.expression.CoerceExpression; +import org.apache.phoenix.expression.Expression; +import org.apache.phoenix.expression.ProjectedColumnExpression; +import org.apache.phoenix.expression.RowKeyColumnExpression; +import org.apache.phoenix.expression.RowValueConstructorExpression; +import org.apache.phoenix.expression.function.FunctionExpression.OrderPreserving; +import org.apache.phoenix.expression.function.ScalarFunction; +import org.apache.phoenix.expression.visitor.StatelessTraverseNoExpressionVisitor; +import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.SortOrder; + +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; + +/** + * Determines if the natural key order of the rows returned by the scan + * will match the order of the expressions. For GROUP BY, if order is preserved we can use + * an optimization during server-side aggregation to do the aggregation on-the-fly versus + * keeping track of each distinct group. We can only do this optimization if all the rows + * for each group will be contiguous. For ORDER BY, we can drop the ORDER BY statement if + * the order is preserved. + * + */ +public class OrderPreservingTracker { + public enum Ordering {ORDERED, UNORDERED}; + + public static class Info { + public final OrderPreserving orderPreserving; + public final int pkPosition; + public final int slotSpan; + + public Info(int pkPosition) { + this.pkPosition = pkPosition; + this.orderPreserving = OrderPreserving.YES; + this.slotSpan = 1; + } + + public Info(Info info, OrderPreserving orderPreserving) { + this.pkPosition = info.pkPosition; + this.slotSpan = info.slotSpan; + this.orderPreserving = orderPreserving; + } + + public Info(Info info, int slotSpan, OrderPreserving orderPreserving) { + this.pkPosition = info.pkPosition; + this.slotSpan = slotSpan; + this.orderPreserving = orderPreserving; + } + } + private final TrackOrderPreservingExpressionVisitor visitor; + private final GroupBy groupBy; + private final Ordering ordering; + private final int pkPositionOffset; + private final List<Info> orderPreservingInfos; + private boolean isOrderPreserving = true; + private Boolean isReverse = null; + + public OrderPreservingTracker(StatementContext context, GroupBy groupBy, Ordering ordering, int nNodes) { + this(context, groupBy, ordering, nNodes, null); + } + + public OrderPreservingTracker(StatementContext context, GroupBy groupBy, Ordering ordering, int nNodes, TupleProjector projector) { + int pkPositionOffset = 0; + if (groupBy.isEmpty()) { // FIXME: would the below table have any of these set in the case of a GROUP BY? + PTable table = context.getResolver().getTables().get(0).getTable(); + boolean isSalted = table.getBucketNum() != null; + boolean isMultiTenant = context.getConnection().getTenantId() != null && table.isMultiTenant(); + boolean isSharedViewIndex = table.getViewIndexId() != null; + // TODO: util for this offset, as it's computed in numerous places + pkPositionOffset = (isSalted ? 1 : 0) + (isMultiTenant ? 1 : 0) + (isSharedViewIndex ? 1 : 0); + } + this.pkPositionOffset = pkPositionOffset; + this.groupBy = groupBy; + this.visitor = new TrackOrderPreservingExpressionVisitor(projector); + this.orderPreservingInfos = Lists.newArrayListWithExpectedSize(nNodes); + this.ordering = ordering; + } + + public void track(Expression node) { + SortOrder sortOrder = node.getSortOrder(); + track(node, sortOrder, sortOrder != SortOrder.getDefault()); + } + + public void track(Expression node, SortOrder sortOrder, boolean isNullsLast) { + if (isOrderPreserving) { + Info info = node.accept(visitor); + if (info == null) { + isOrderPreserving = false; + } else { + // If the expression is sorted in a different order than the specified sort order + // then the expressions are not order preserving. + if (node.getSortOrder() != sortOrder) { + if (isReverse == null) { + isReverse = true; + /* + * When a GROUP BY is not order preserving, we cannot do a reverse + * scan to eliminate the ORDER BY since our server-side scan is not + * ordered in that case. + */ + if (!groupBy.isEmpty() && !groupBy.isOrderPreserving()) { + isOrderPreserving = false; + return; + } + } else if (!isReverse){ + isOrderPreserving = false; + return; + } + } else { + if (isReverse == null) { + isReverse = false; + } else if (isReverse){ + isOrderPreserving = false; + return; + } + } + if (node.isNullable()) { + if (!Boolean.valueOf(isNullsLast).equals(isReverse)) { + isOrderPreserving = false; + return; + } + } + orderPreservingInfos.add(info); + } + } + } + + public boolean isOrderPreserving() { + if (!isOrderPreserving) { + return false; + } + if (ordering == Ordering.UNORDERED) { + // Sort by position + Collections.sort(orderPreservingInfos, new Comparator<Info>() { + @Override + public int compare(Info o1, Info o2) { + return o1.pkPosition-o2.pkPosition; + } + }); + } + // Determine if there are any gaps in the PK columns (in which case we don't need + // to sort in the coprocessor because the keys will already naturally be in sorted + // order. + int prevSlotSpan = 1; + int prevPos = pkPositionOffset - 1; + OrderPreserving prevOrderPreserving = OrderPreserving.YES; + for (int i = 0; i < orderPreservingInfos.size() && isOrderPreserving; i++) { + Info entry = orderPreservingInfos.get(i); + int pos = entry.pkPosition; + isOrderPreserving &= (entry.orderPreserving != OrderPreserving.NO) && (pos == prevPos || ((pos - prevSlotSpan == prevPos) && (prevOrderPreserving == OrderPreserving.YES))); + prevPos = pos; + prevSlotSpan = entry.slotSpan; + prevOrderPreserving = entry.orderPreserving; + } + return isOrderPreserving; + } + + public boolean isReverse() { + return Boolean.TRUE.equals(isReverse); + } + + private static class TrackOrderPreservingExpressionVisitor extends StatelessTraverseNoExpressionVisitor<Info> { + private final TupleProjector projector; + + public TrackOrderPreservingExpressionVisitor(TupleProjector projector) { + this.projector = projector; + } + + @Override + public Info visit(RowKeyColumnExpression node) { + return new Info(node.getPosition()); + } + + @Override + public Info visit(ProjectedColumnExpression node) { + if (projector == null) { + return super.visit(node); + } + Expression expression = projector.getExpressions()[node.getPosition()]; + // FIXME: prevents infinite recursion for union all in subquery, but + // should a ProjectedColumnExpression be used in this case? Wouldn't + // it make more sense to not create this wrapper in this case? + if (expression == node) { + return super.visit(node); + } + return expression.accept(this); + } + + @Override + public Iterator<Expression> visitEnter(ScalarFunction node) { + return node.preservesOrder() == OrderPreserving.NO ? Iterators.<Expression> emptyIterator() : Iterators + .singletonIterator(node.getChildren().get(node.getKeyFormationTraversalIndex())); + } + + @Override + public Info visitLeave(ScalarFunction node, List<Info> l) { + if (l.isEmpty()) { return null; } + Info info = l.get(0); + // Keep the minimum value between this function and the current value, + // so that we never increase OrderPreserving from NO or YES_IF_LAST. + OrderPreserving orderPreserving = OrderPreserving.values()[Math.min(node.preservesOrder().ordinal(), info.orderPreserving.ordinal())]; + if (orderPreserving == info.orderPreserving) { + return info; + } + return new Info(info, orderPreserving); + } + + @Override + public Iterator<Expression> visitEnter(CoerceExpression node) { + return node.getChildren().iterator(); + } + + @Override + public Info visitLeave(CoerceExpression node, List<Info> l) { + if (l.isEmpty()) { return null; } + return l.get(0); + } + + @Override + public Iterator<Expression> visitEnter(RowValueConstructorExpression node) { + return node.getChildren().iterator(); + } + + @Override + public Info visitLeave(RowValueConstructorExpression node, List<Info> l) { + // Child expression returned null and was filtered, so not order preserving + if (l.size() != node.getChildren().size()) { return null; } + Info firstInfo = l.get(0); + Info lastInfo = firstInfo; + // Check that pkPos are consecutive which is the only way a RVC can be order preserving + for (int i = 1; i < l.size(); i++) { + // not order preserving since it's not last + if (lastInfo.orderPreserving == OrderPreserving.YES_IF_LAST) { return null; } + Info info = l.get(i); + // not order preserving since there's a gap in the pk + if (info.pkPosition != lastInfo.pkPosition + 1) { return null; } + lastInfo = info; + } + return new Info(firstInfo, l.size(), lastInfo.orderPreserving); + } + } +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/b4f0f167/phoenix-core/src/main/java/org/apache/phoenix/compile/TrackOrderPreservingExpressionCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/TrackOrderPreservingExpressionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/TrackOrderPreservingExpressionCompiler.java deleted file mode 100644 index 9fd6837..0000000 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/TrackOrderPreservingExpressionCompiler.java +++ /dev/null @@ -1,249 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.phoenix.compile; - -import java.sql.SQLException; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; - -import org.apache.phoenix.compile.GroupByCompiler.GroupBy; -import org.apache.phoenix.execute.TupleProjector; -import org.apache.phoenix.expression.Expression; -import org.apache.phoenix.expression.LiteralExpression; -import org.apache.phoenix.expression.RowKeyColumnExpression; -import org.apache.phoenix.expression.function.FunctionExpression; -import org.apache.phoenix.expression.function.FunctionExpression.OrderPreserving; -import org.apache.phoenix.parse.CaseParseNode; -import org.apache.phoenix.parse.ColumnParseNode; -import org.apache.phoenix.parse.DivideParseNode; -import org.apache.phoenix.parse.MultiplyParseNode; -import org.apache.phoenix.parse.SubtractParseNode; -import org.apache.phoenix.schema.ColumnRef; -import org.apache.phoenix.schema.PTable; -import org.apache.phoenix.schema.PTableType; -import org.apache.phoenix.schema.SortOrder; -import com.google.common.collect.Lists; - -/** - * Visitor that builds the expressions of a GROUP BY and ORDER BY clause. While traversing - * the parse node tree, the visitor also determines if the natural key order of the scan - * will match the order of the expressions. For GROUP BY, if order is preserved we can use - * an optimization during server-side aggregation to do the aggregation on-the-fly versus - * keeping track of each distinct group. We can only do this optimization if all the rows - * for each group will be contiguous. For ORDER BY, we can drop the ORDER BY statement if - * the order is preserved. - * - */ -public class TrackOrderPreservingExpressionCompiler extends ExpressionCompiler { - public enum Ordering {ORDERED, UNORDERED}; - - private final List<Entry> entries; - private final Ordering ordering; - private final int positionOffset; - private final TupleProjector tupleProjector; // for derived-table query compilation - private OrderPreserving orderPreserving = OrderPreserving.YES; - private ColumnRef columnRef; - private boolean isOrderPreserving = true; - private Boolean isReverse; - - TrackOrderPreservingExpressionCompiler(StatementContext context, GroupBy groupBy, int expectedEntrySize, Ordering ordering, TupleProjector tupleProjector) { - super(context, groupBy); - PTable table = context.getResolver().getTables().get(0).getTable(); - boolean isSalted = table.getBucketNum() != null; - boolean isMultiTenant = context.getConnection().getTenantId() != null && table.isMultiTenant(); - boolean isSharedViewIndex = table.getViewIndexId() != null; - // TODO: util for this offset, as it's computed in numerous places - positionOffset = (isSalted ? 1 : 0) + (isMultiTenant ? 1 : 0) + (isSharedViewIndex ? 1 : 0); - entries = Lists.newArrayListWithExpectedSize(expectedEntrySize); - this.ordering = ordering; - this.tupleProjector = tupleProjector; - } - - public Boolean isReverse() { - return isReverse; - } - - public boolean isOrderPreserving() { - if (!isOrderPreserving) { - return false; - } - if (ordering == Ordering.UNORDERED) { - // Sort by position - Collections.sort(entries, new Comparator<Entry>() { - @Override - public int compare(Entry o1, Entry o2) { - return o1.getPkPosition()-o2.getPkPosition(); - } - }); - } - // Determine if there are any gaps in the PK columns (in which case we don't need - // to sort in the coprocessor because the keys will already naturally be in sorted - // order. - int prevPos = positionOffset - 1; - OrderPreserving prevOrderPreserving = OrderPreserving.YES; - for (int i = 0; i < entries.size() && isOrderPreserving; i++) { - Entry entry = entries.get(i); - int pos = entry.getPkPosition(); - isOrderPreserving &= (entry.getOrderPreserving() != OrderPreserving.NO) && (pos == prevPos || ((pos - 1 == prevPos) && (prevOrderPreserving == OrderPreserving.YES))); - prevPos = pos; - prevOrderPreserving = entries.get(i).getOrderPreserving(); - } - return isOrderPreserving; - } - - @Override - protected Expression addExpression(Expression expression) { - // TODO: have FunctionExpression visitor instead and remove this cast - if (expression instanceof FunctionExpression) { - // Keep the minimum value between this function and the current value, - // so that we never increase OrderPreserving from NO or YES_IF_LAST. - orderPreserving = OrderPreserving.values()[Math.min(orderPreserving.ordinal(), ((FunctionExpression)expression).preservesOrder().ordinal())]; - } - return super.addExpression(expression); - } - - @Override - public boolean visitEnter(CaseParseNode node) throws SQLException { - orderPreserving = OrderPreserving.NO; - return super.visitEnter(node); - } - - @Override - public boolean visitEnter(DivideParseNode node) throws SQLException { - // A divide expression may not preserve row order. - // For example: GROUP BY 1/x - orderPreserving = OrderPreserving.NO; - return super.visitEnter(node); - } - - @Override - public boolean visitEnter(SubtractParseNode node) throws SQLException { - // A subtract expression may not preserve row order. - // For example: GROUP BY 10 - x - orderPreserving = OrderPreserving.NO; - return super.visitEnter(node); - } - - @Override - public boolean visitEnter(MultiplyParseNode node) throws SQLException { - // A multiply expression may not preserve row order. - // For example: GROUP BY -1 * x - orderPreserving = OrderPreserving.NO; - return super.visitEnter(node); - } - - @Override - public void reset() { - super.reset(); - columnRef = null; - orderPreserving = OrderPreserving.YES; - } - - @Override - protected ColumnRef resolveColumn(ColumnParseNode node) throws SQLException { - ColumnRef ref = super.resolveColumn(node); - // If we encounter any non PK column, then we can't aggregate on-the-fly - // because the distinct groups have no correlation to the KV column value - if (getColumnPKPosition(ref) < 0) { - orderPreserving = OrderPreserving.NO; - } - - if (columnRef == null) { - columnRef = ref; - } else if (!columnRef.equals(ref)) { - // If we encounter more than one column reference in an expression, - // we can't assume the result of the expression will be key ordered. - // For example GROUP BY a * b - orderPreserving = OrderPreserving.NO; - } - return ref; - } - - private int getColumnPKPosition(ColumnRef ref) { - if (tupleProjector != null && ref.getTable().getType() == PTableType.SUBQUERY) { - Expression expression = tupleProjector.getExpressions()[ref.getColumnPosition()]; - if (expression instanceof RowKeyColumnExpression) { - return ((RowKeyColumnExpression) expression).getPosition(); - } - } - - return ref.getPKSlotPosition(); - } - - public boolean addEntry(Expression expression) { - if (expression instanceof LiteralExpression) { - return false; - } - isOrderPreserving &= (orderPreserving != OrderPreserving.NO); - entries.add(new Entry(expression, columnRef, orderPreserving)); - return true; - } - - public boolean addEntry(Expression expression, SortOrder sortOrder) { - // If the expression is sorted in a different order than the specified sort order - // then the expressions are not order preserving. - if (expression.getSortOrder() != sortOrder) { - if (isReverse == null) { - isReverse = true; - } else if (!isReverse){ - orderPreserving = OrderPreserving.NO; - } - } else { - if (isReverse == null) { - isReverse = false; - } else if (isReverse){ - orderPreserving = OrderPreserving.NO; - } - } - return addEntry(expression); - } - - public List<Entry> getEntries() { - return entries; - } - - public class Entry { - private final Expression expression; - private final ColumnRef columnRef; - private final OrderPreserving orderPreserving; - - private Entry(Expression expression, ColumnRef columnRef, OrderPreserving orderPreserving) { - this.expression = expression; - this.columnRef = columnRef; - this.orderPreserving = orderPreserving; - } - - public Expression getExpression() { - return expression; - } - - public int getPkPosition() { - return getColumnPKPosition(columnRef); - } - - public int getColumnPosition() { - return columnRef.getColumnPosition(); - } - - public OrderPreserving getOrderPreserving() { - return orderPreserving; - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/b4f0f167/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java index 77c1f9e..77eb237 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java @@ -41,6 +41,7 @@ import java.util.Properties; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.compile.OrderByCompiler.OrderBy; import org.apache.phoenix.coprocessor.BaseScannerRegionObserver; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.expression.Expression; @@ -52,6 +53,7 @@ import org.apache.phoenix.expression.function.TimeUnit; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.jdbc.PhoenixPreparedStatement; +import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.query.BaseConnectionlessQueryTest; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.AmbiguousColumnException; @@ -1450,8 +1452,8 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest { conn.createStatement().execute("CREATE TABLE t (k1 varchar, k2 varchar, v varchar, constraint pk primary key(k1,k2))"); ResultSet rs; String[] queries = { - "SELECT DISTINCT v FROM T ORDER BY v LIMIT 3", - "SELECT v FROM T GROUP BY v,k1 ORDER BY v LIMIT 3", +// "SELECT DISTINCT v FROM T ORDER BY v LIMIT 3", +// "SELECT v FROM T GROUP BY v,k1 ORDER BY v LIMIT 3", "SELECT DISTINCT count(*) FROM T GROUP BY k1 LIMIT 3", "SELECT count(1) FROM T GROUP BY v,k1 LIMIT 3", "SELECT max(v) FROM T GROUP BY k1,k2 HAVING count(k1) > 1 LIMIT 3", @@ -1461,7 +1463,8 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest { for (int i = 0; i < queries.length; i++) { query = queries[i]; rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertFalse("Did not expected to find GROUP BY limit optimization in: " + query, QueryUtil.getExplainPlan(rs).contains(" LIMIT 3 GROUPS")); + String explainPlan = QueryUtil.getExplainPlan(rs); + assertFalse("Did not expected to find GROUP BY limit optimization in: " + query, explainPlan.contains(" LIMIT 3 GROUPS")); } } @@ -1631,4 +1634,103 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest { } + @Test + public void testOrderByOrderPreservingFwd() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + conn.createStatement().execute("CREATE TABLE t (k1 date not null, k2 date not null, k3 date not null, v varchar, constraint pk primary key(k1,k2,k3))"); + String[] queries = { + "SELECT * FROM T ORDER BY (k1,k2), k3", + "SELECT * FROM T ORDER BY k1,k2,k3", + "SELECT * FROM T ORDER BY k1,k2", + "SELECT * FROM T ORDER BY k1", + "SELECT * FROM T ORDER BY CAST(k1 AS TIMESTAMP)", + "SELECT * FROM T ORDER BY (k1,k2,k3)", + "SELECT * FROM T ORDER BY TRUNC(k1, 'DAY'), CEIL(k2, 'HOUR')", + "SELECT * FROM T ORDER BY INVERT(k1) DESC", + }; + String query; + for (int i = 0; i < queries.length; i++) { + query = queries[i]; + QueryPlan plan = conn.createStatement().unwrap(PhoenixStatement.class).compileQuery(query); + assertTrue("Expected order by to be compiled out: " + query, plan.getOrderBy() == OrderBy.FWD_ROW_KEY_ORDER_BY); + } + } + + @Test + public void testOrderByOrderPreservingRev() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + conn.createStatement().execute("CREATE TABLE t (k1 date not null, k2 date not null, k3 date not null, v varchar, constraint pk primary key(k1,k2 DESC,k3))"); + String[] queries = { + "SELECT * FROM T ORDER BY INVERT(k1),k2", + "SELECT * FROM T ORDER BY INVERT(k1)", + "SELECT * FROM T ORDER BY TRUNC(k1, 'DAY') DESC, CEIL(k2, 'HOUR') DESC", + "SELECT * FROM T ORDER BY k1 DESC", + }; + String query; + for (int i = 0; i < queries.length; i++) { + query = queries[i]; + QueryPlan plan = conn.createStatement().unwrap(PhoenixStatement.class).compileQuery(query); + assertTrue("Expected order by to be compiled out: " + query, plan.getOrderBy() == OrderBy.REV_ROW_KEY_ORDER_BY); + } + } + + @Test + public void testNotOrderByOrderPreserving() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + conn.createStatement().execute("CREATE TABLE t (k1 date not null, k2 date not null, k3 date not null, v varchar, constraint pk primary key(k1,k2,k3))"); + String[] queries = { + "SELECT * FROM T ORDER BY k1,k3", + "SELECT * FROM T ORDER BY SUBSTR(TO_CHAR(k1),1,4)", + "SELECT * FROM T ORDER BY k2", + "SELECT * FROM T ORDER BY INVERT(k1),k3", + "SELECT * FROM T ORDER BY CASE WHEN k1 = CURRENT_DATE() THEN 0 ELSE 1 END", + "SELECT * FROM T ORDER BY TO_CHAR(k1)", + }; + String query; + for (int i = 0; i < queries.length; i++) { + query = queries[i]; + QueryPlan plan = conn.createStatement().unwrap(PhoenixStatement.class).compileQuery(query); + assertFalse("Expected order by not to be compiled out: " + query, plan.getOrderBy().getOrderByExpressions().isEmpty()); + } + } + + @Test + public void testGroupByOrderPreserving() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + conn.createStatement().execute("CREATE TABLE t (k1 date not null, k2 date not null, k3 date not null, v varchar, constraint pk primary key(k1,k2,k3))"); + String[] queries = { + "SELECT 1 FROM T GROUP BY k3, (k1,k2)", + "SELECT 1 FROM T GROUP BY k2,k1,k3", + "SELECT 1 FROM T GROUP BY k1,k2", + "SELECT 1 FROM T GROUP BY k1", + "SELECT 1 FROM T GROUP BY CAST(k1 AS TIMESTAMP)", + "SELECT 1 FROM T GROUP BY (k1,k2,k3)", + "SELECT 1 FROM T GROUP BY TRUNC(k2, 'DAY'), CEIL(k1, 'HOUR')", + }; + String query; + for (int i = 0; i < queries.length; i++) { + query = queries[i]; + QueryPlan plan = conn.createStatement().unwrap(PhoenixStatement.class).compileQuery(query); + assertTrue("Expected group by to be order preserving: " + query, plan.getGroupBy().isOrderPreserving()); + } + } + + @Test + public void testNotGroupByOrderPreserving() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + conn.createStatement().execute("CREATE TABLE t (k1 date not null, k2 date not null, k3 date not null, v varchar, constraint pk primary key(k1,k2,k3))"); + String[] queries = { + "SELECT 1 FROM T GROUP BY k1,k3", + "SELECT 1 FROM T GROUP BY k2", + "SELECT 1 FROM T GROUP BY INVERT(k1),k3", + "SELECT 1 FROM T GROUP BY CASE WHEN k1 = CURRENT_DATE() THEN 0 ELSE 1 END", + "SELECT 1 FROM T GROUP BY TO_CHAR(k1)", + }; + String query; + for (int i = 0; i < queries.length; i++) { + query = queries[i]; + QueryPlan plan = conn.createStatement().unwrap(PhoenixStatement.class).compileQuery(query); + assertFalse("Expected group by not to be order preserving: " + query, plan.getGroupBy().isOrderPreserving()); + } + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/b4f0f167/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java index 67c44bd..cd51683 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java @@ -19,6 +19,7 @@ package org.apache.phoenix.compile; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import java.sql.Array; import java.sql.Connection; @@ -33,7 +34,6 @@ import java.util.Properties; import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.compile.OrderByCompiler.OrderBy; -import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.query.BaseConnectionlessQueryTest; @@ -84,9 +84,8 @@ public class QueryOptimizerTest extends BaseConnectionlessQueryTest { try{ conn.createStatement().execute("CREATE TABLE foo (k VARCHAR NOT NULL PRIMARY KEY, v VARCHAR) IMMUTABLE_ROWS=true"); PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class); - QueryPlan plan = stmt.optimizeQuery("SELECT * FROM foo ORDER BY 1,2,3"); - } catch (SQLException e) { - assertEquals(SQLExceptionCode.PARAM_INDEX_OUT_OF_BOUND.getErrorCode(), e.getErrorCode()); + QueryPlan plan = stmt.optimizeQuery("SELECT * FROM foo ORDER BY 'a','b','c'"); + assertTrue(plan.getOrderBy().getOrderByExpressions().isEmpty()); } finally { conn.close(); }