This is an automated email from the ASF dual-hosted git repository. kadir pushed a commit to branch 4.x in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x by this push: new c1aa9f1 PHOENIX-6458 Using global indexes for queries with uncovered columns (#1256) c1aa9f1 is described below commit c1aa9f1b18ea8be94aa96bce04fd57e2ec715aaf Author: kadirozde <37155482+kadiro...@users.noreply.github.com> AuthorDate: Thu Feb 24 14:56:24 2022 -0800 PHOENIX-6458 Using global indexes for queries with uncovered columns (#1256) --- .../end2end/index/GlobalIndexCheckerIT.java | 122 +++++++++++++++++++++ .../end2end/index/GlobalIndexOptimizationIT.java | 78 ++++++------- .../apache/phoenix/compile/ExpressionCompiler.java | 11 +- .../org/apache/phoenix/compile/FromCompiler.java | 14 ++- .../org/apache/phoenix/compile/JoinCompiler.java | 11 +- .../apache/phoenix/compile/ProjectionCompiler.java | 17 ++- .../apache/phoenix/compile/StatementContext.java | 9 ++ .../phoenix/compile/TupleProjectionCompiler.java | 23 ++-- .../org/apache/phoenix/compile/WhereCompiler.java | 9 +- .../coprocessor/BaseScannerRegionObserver.java | 7 ++ .../GroupedAggregateRegionObserver.java | 12 +- .../org/apache/phoenix/execute/BaseQueryPlan.java | 27 +++-- .../phoenix/iterate/BaseResultIterators.java | 7 +- .../org/apache/phoenix/iterate/ExplainTable.java | 6 +- .../iterate/NonAggregateRegionScannerFactory.java | 24 ++-- .../phoenix/iterate/RegionScannerFactory.java | 23 ++-- .../apache/phoenix/optimize/QueryOptimizer.java | 1 + ...xDataColumnRef.java => IndexDataColumnRef.java} | 41 +++++-- .../java/org/apache/phoenix/schema/TableRef.java | 32 ++++-- .../java/org/apache/phoenix/util/IndexUtil.java | 62 +++++++++-- .../java/org/apache/phoenix/util/ScanUtil.java | 11 ++ 21 files changed, 403 insertions(+), 144 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java index cb01149..02c57e2 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java @@ -306,6 +306,44 @@ public class GlobalIndexCheckerIT extends BaseTest { assertEquals("ae", rs.getString(1)); assertEquals("efg", rs.getString(2)); assertFalse(rs.next()); + conn.createStatement().execute("DROP INDEX " + indexTableName + " on " + + dataTableName); + // Run the previous test on an uncovered global index + indexTableName = generateUniqueName(); + conn.createStatement().execute("CREATE INDEX " + indexTableName + " on " + + dataTableName + " (PHOENIX_ROW_TIMESTAMP())" + + (async ? "ASYNC" : "")+ this.indexDDLOptions); + if (async) { + // Run the index MR job to rebuild the index and verify that index is built correctly + IndexToolIT.runIndexTool(false, null, dataTableName, + indexTableName, null, 0, IndexTool.IndexVerifyType.AFTER); + } + // Verify that without hint, the index table is not selected + assertIndexTableNotSelected(conn, dataTableName, indexTableName, query); + + // Verify that we will read from the index table with the index hint + query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ " + + "val1, val2, PHOENIX_ROW_TIMESTAMP() from " + dataTableName + " WHERE " + + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + initial.toString() + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; + + assertExplainPlan(conn, query, dataTableName, indexTableName); + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("ab", rs.getString(1)); + assertEquals("abc", rs.getString(2)); + assertTrue(rs.next()); + assertEquals("bc", rs.getString(1)); + assertEquals("bcd", rs.getString(2)); + assertTrue(rs.next()); + assertEquals("bc", rs.getString(1)); + assertEquals("ccc", rs.getString(2)); + assertTrue(rs.next()); + assertEquals("de", rs.getString(1)); + assertEquals("def", rs.getString(2)); + assertTrue(rs.next()); + assertEquals("ae", rs.getString(1)); + assertEquals("efg", rs.getString(2)); + assertFalse(rs.next()); } } @@ -430,6 +468,90 @@ public class GlobalIndexCheckerIT extends BaseTest { } } + private void assertIndexTableNotSelected(Connection conn, String dataTableName, String indexTableName, String sql) + throws Exception { + try { + assertExplainPlan(conn, sql, dataTableName, indexTableName); + throw new AssertionError("The index table should not be selected without an index hint"); + } catch (AssertionError error){ + //expected + } + } + + @Test + public void testUncoveredGlobalIndex() throws Exception { + if (async) { + return; + } + String dataTableName = generateUniqueName(); + populateTable(dataTableName); // with two rows ('a', 'ab', 'abc', 'abcd') and ('b', 'bc', 'bcd', 'bcde') + try (Connection conn = DriverManager.getConnection(getUrl())) { + String indexTableName = generateUniqueName(); + conn.createStatement().execute("CREATE INDEX " + indexTableName + " on " + + dataTableName + " (val1) include (val2)" + this.indexDDLOptions); + // Verify that without hint, the index table is not selected + assertIndexTableNotSelected(conn, dataTableName, indexTableName, + "SELECT val3 from " + dataTableName + " WHERE val1 = 'bc' AND (val2 = 'bcd' OR val3 ='bcde')"); + + //Verify that with index hint, we will read from the index table even though val3 is not included by the index table + String selectSql = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ val3 from " + + dataTableName + " WHERE val1 = 'bc' AND (val2 = 'bcd' OR val3 ='bcde')"; + assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + ResultSet rs = conn.createStatement().executeQuery(selectSql); + assertTrue(rs.next()); + assertEquals("bcde", rs.getString(1)); + assertFalse(rs.next()); + conn.createStatement().execute("DROP INDEX " + indexTableName + " on " + dataTableName); + // Create an index does not include any columns + indexTableName = generateUniqueName(); + conn.createStatement().execute("CREATE INDEX " + indexTableName + " on " + + dataTableName + " (val1)" + this.indexDDLOptions); + conn.commit(); + + // Verify that without hint, the index table is not selected + assertIndexTableNotSelected(conn, dataTableName, indexTableName, + "SELECT id from " + dataTableName + " WHERE val1 = 'bc' AND (val2 = 'bcd' OR val3 ='bcde')"); + selectSql = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ id from " + dataTableName + " WHERE val1 = 'bc' AND (val2 = 'bcd' OR val3 ='bcde')"; + //Verify that we will read from the index table + assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + rs = conn.createStatement().executeQuery(selectSql); + assertTrue(rs.next()); + assertEquals("b", rs.getString(1)); + assertFalse(rs.next()); + + // Add another row and run a group by query where the uncovered index should be used + conn.createStatement().execute("upsert into " + dataTableName + " (id, val1, val2, val3) values ('c', 'ab','cde', 'cdef')"); + conn.commit(); + // Verify that without hint, the index table is not selected + assertIndexTableNotSelected(conn, dataTableName, indexTableName, + "SELECT count(*) from " + dataTableName + " where val1 > '0' GROUP BY val1"); + selectSql = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ count(*) from " + dataTableName + " where val1 > '0' GROUP BY val1"; + //Verify that we will read from the index table + assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + rs = conn.createStatement().executeQuery(selectSql); + assertTrue(rs.next()); + assertEquals(2, rs.getInt(1)); + assertTrue(rs.next()); + assertEquals(1, rs.getInt(1)); + assertFalse(rs.next()); + // Run an order by query where the uncovered index should be used + // Verify that without hint, the index table is not selected + assertIndexTableNotSelected(conn, dataTableName, indexTableName, + "SELECT val3 from " + dataTableName + " where val1 > '0' ORDER BY val1"); + selectSql = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ val3 from " + dataTableName + " where val1 > '0' ORDER BY val1"; + //Verify that we will read from the index table + assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + rs = conn.createStatement().executeQuery(selectSql); + assertTrue(rs.next()); + assertEquals("abcd", rs.getString(1)); + assertTrue(rs.next()); + assertEquals("cdef", rs.getString(1)); + assertTrue(rs.next()); + assertEquals("bcde", rs.getString(1)); + assertFalse(rs.next()); + } + } + @Test public void testSimulateConcurrentUpdates() throws Exception { try (Connection conn = DriverManager.getConnection(getUrl())) { diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java index 96e83f5..99f5997 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java @@ -230,17 +230,12 @@ public class GlobalIndexOptimizationIT extends ParallelStatsDisabledIT { query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ t_id, k1, k2, k3, V1 from " + dataTableFullName + " where v1<='z' and k3 > 1 order by V1,t_id"; rs = conn1.createStatement().executeQuery("EXPLAIN " + query); - expected = - "CLIENT PARALLEL \\d-WAY FULL SCAN OVER " + dataTableName + "\n" + - " SERVER FILTER BY K3 > 1\n" + - " SERVER SORTED BY \\[" + dataTableName + ".V1, " + dataTableName + ".T_ID\\]\n" + - "CLIENT MERGE SORT\n" + - " SKIP-SCAN-JOIN TABLE 0\n" + - " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName + " \\[\\*\\] - \\['z'\\]\n" + - " SERVER FILTER BY FIRST KEY ONLY\n" + - " DYNAMIC SERVER FILTER BY \\(\"" + dataTableName + ".T_ID\", \"" + dataTableName + ".K1\", \"" + dataTableName + ".K2\"\\) IN \\(\\(\\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+\\)\\)"; + expected = + "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName + " [*] - ['z']\n"+ + " SERVER MERGE [0.K3]\n" + + " SERVER FILTER BY FIRST KEY ONLY AND \"K3\" > 1"; actual = QueryUtil.getExplainPlan(rs); - assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, Pattern.matches(expected, actual)); + assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, actual.equals(expected)); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -265,17 +260,15 @@ public class GlobalIndexOptimizationIT extends ParallelStatsDisabledIT { query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ t_id, V1, k3 from " + dataTableFullName + " where v1 <='z' group by v1,t_id, k3"; rs = conn1.createStatement().executeQuery("EXPLAIN " + query); - - expected = - "CLIENT PARALLEL \\d-WAY FULL SCAN OVER " + dataTableName + "\n" + - " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[" + dataTableName + ".V1, " + dataTableName + ".T_ID, " + dataTableName + ".K3\\]\n" + - "CLIENT MERGE SORT\n" + - " SKIP-SCAN-JOIN TABLE 0\n" + - " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName + " \\[\\*\\] - \\['z'\\]\n" + - " SERVER FILTER BY FIRST KEY ONLY\n" + - " DYNAMIC SERVER FILTER BY \\(\"" + dataTableName + ".T_ID\", \"" + dataTableName + ".K1\", \"" + dataTableName + ".K2\"\\) IN \\(\\(\\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+\\)\\)"; + expected = + "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName + " [*] - ['z']\n"+ + " SERVER MERGE [0.K3]\n" + + " SERVER FILTER BY FIRST KEY ONLY\n" + + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"V1\", \"T_ID\", \"K3\"]\n"+ + "CLIENT MERGE SORT"; + actual = QueryUtil.getExplainPlan(rs); - assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, Pattern.matches(expected, actual)); + assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, actual.equals(expected)); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -299,16 +292,13 @@ public class GlobalIndexOptimizationIT extends ParallelStatsDisabledIT { query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ v1,sum(k3) from " + dataTableFullName + " where v1 <='z' group by v1 order by v1"; rs = conn1.createStatement().executeQuery("EXPLAIN " + query); - expected = - "CLIENT PARALLEL \\d-WAY FULL SCAN OVER " + dataTableName + "\n" + - " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[" + dataTableName + ".V1\\]\n" + - "CLIENT MERGE SORT\n" + - " SKIP-SCAN-JOIN TABLE 0\n" + - " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName + " \\[\\*\\] - \\['z'\\]\n" + - " SERVER FILTER BY FIRST KEY ONLY\n" + - " DYNAMIC SERVER FILTER BY \\(\"" + dataTableName + ".T_ID\", \"" + dataTableName + ".K1\", \"" + dataTableName + ".K2\"\\) IN \\(\\(\\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+\\)\\)"; + expected = + "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName + " [*] - ['z']\n"+ + " SERVER MERGE [0.K3]\n" + + " SERVER FILTER BY FIRST KEY ONLY\n" + + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"V1\"]"; actual = QueryUtil.getExplainPlan(rs); - assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, Pattern.matches(expected, actual)); + assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, actual.equals(expected)); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -340,12 +330,10 @@ public class GlobalIndexOptimizationIT extends ParallelStatsDisabledIT { ResultSet rs = conn1.createStatement().executeQuery("EXPLAIN "+ query); String actual = QueryUtil.getExplainPlan(rs); - String expected = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + dataTableName + " \\['tid1'\\]\n" + - " SKIP-SCAN-JOIN TABLE 0\n" + - " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName + " \\['tid1','a'\\]\n" + - " SERVER FILTER BY FIRST KEY ONLY\n" + - " DYNAMIC SERVER FILTER BY \\(\"" + dataTableName + ".K1\", \"" + dataTableName + ".K2\"\\) IN \\(\\(\\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+\\)\\)"; - assertTrue("Expected:\n" + expected + "\ndid not match\n" + actual, Pattern.matches(expected, actual)); + String expected = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName + " ['tid1','a']\n" + + " SERVER MERGE [0.K3]\n" + + " SERVER FILTER BY FIRST KEY ONLY"; + assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, actual.equals(expected)); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -393,15 +381,15 @@ public class GlobalIndexOptimizationIT extends ParallelStatsDisabledIT { * This inner "_IDX_" + dataTableName use skipScan, and all the whereExpressions are already in SkipScanFilter, * so there is no other RowKeyComparisonFilter needed. */ + String actual = QueryUtil.getExplainPlan(rs); String expected = - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + dataTableName + "\n" + - " SERVER FILTER BY V1 = 'a'\n" + - " SKIP-SCAN-JOIN TABLE 0\n" + - " CLIENT PARALLEL 1-WAY SKIP SCAN ON 2 KEYS OVER _IDX_" + dataTableName + " \\[" + Short.MIN_VALUE + ",1\\] - \\[" + Short.MIN_VALUE + ",2\\]\n" + - " SERVER FILTER BY FIRST KEY ONLY\n" + - " DYNAMIC SERVER FILTER BY \\(\"" + viewName + ".T_ID\", \"" + viewName + ".K1\", \"" + viewName + ".K2\"\\) IN \\(\\(\\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+\\)\\)"; - assertTrue("Expected:\n" + expected + "\ndid not match\n" + actual, Pattern.matches(expected,actual)); + "CLIENT PARALLEL 1-WAY SKIP SCAN ON 2 KEYS OVER _IDX_" + dataTableName + + " [" + Short.MIN_VALUE + ",1] - [" + Short.MIN_VALUE + ",2]\n" + + " SERVER MERGE [0.K3]\n" + + " SERVER FILTER BY FIRST KEY ONLY"; + + assertEquals(expected,actual); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -409,7 +397,7 @@ public class GlobalIndexOptimizationIT extends ParallelStatsDisabledIT { assertEquals(2, rs.getInt("k1")); assertEquals(4, rs.getInt("k2")); assertEquals(2, rs.getInt("k3")); - assertEquals("a", rs.getString("v1")); + assertEquals("a", rs.getString(5)); //TODO use name v1 instead of position 5, see PHOENIX-6644 assertFalse(rs.next()); } finally { conn1.close(); @@ -474,7 +462,7 @@ public class GlobalIndexOptimizationIT extends ParallelStatsDisabledIT { assertFalse(rs.next()); // No where clause - query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ t_id, k1, k2, k3, V1 from " + dataTableFullName + " order by V1,t_id"; + query = "SELECT t_id, k1, k2, k3, V1 from " + dataTableFullName + " order by V1,t_id"; rs = conn1.createStatement().executeQuery("EXPLAIN " + query); assertEquals( @@ -511,7 +499,7 @@ public class GlobalIndexOptimizationIT extends ParallelStatsDisabledIT { assertFalse(rs.next()); // No where clause in index scan - query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ t_id, k1, k2, k3, V1 from " + dataTableFullName + " where k3 > 1 order by V1,t_id"; + query = "SELECT t_id, k1, k2, k3, V1 from " + dataTableFullName + " where k3 > 1 order by V1,t_id"; rs = conn1.createStatement().executeQuery("EXPLAIN " + query); assertEquals( diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java index d1b44df..986cdf3 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java @@ -112,7 +112,7 @@ import org.apache.phoenix.schema.ColumnFamilyNotFoundException; import org.apache.phoenix.schema.ColumnNotFoundException; import org.apache.phoenix.schema.ColumnRef; import org.apache.phoenix.schema.DelegateDatum; -import org.apache.phoenix.schema.LocalIndexDataColumnRef; +import org.apache.phoenix.schema.IndexDataColumnRef; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PDatum; import org.apache.phoenix.schema.PTable; @@ -140,6 +140,8 @@ import org.apache.phoenix.util.IndexUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.StringUtil; +import static org.apache.phoenix.util.IndexUtil.isHintedGlobalIndex; + public class ExpressionCompiler extends UnsupportedAllParseNodeVisitor<Expression> { private boolean isAggregate; @@ -371,9 +373,12 @@ public class ExpressionCompiler extends UnsupportedAllParseNodeVisitor<Expressio // Rather than not use a local index when a column not contained by it is referenced, we // join back to the data table in our coprocessor since this is a relatively cheap // operation given that we know the join is local. - if (context.getCurrentTable().getTable().getIndexType() == IndexType.LOCAL) { + if (context.getCurrentTable().getTable().getIndexType() == IndexType.LOCAL + || isHintedGlobalIndex(context.getCurrentTable())) { try { - return new LocalIndexDataColumnRef(context, context.getCurrentTable(), node.getName()); + context.setUncoveredIndex(true); + return new IndexDataColumnRef(context, context.getCurrentTable(), + node.getName()); } catch (ColumnFamilyNotFoundException c) { throw e; } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java index 197a0a6..88d80aa 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java @@ -103,6 +103,7 @@ import org.apache.phoenix.thirdparty.com.google.common.collect.ListMultimap; import org.apache.phoenix.thirdparty.com.google.common.collect.Lists; import static org.apache.phoenix.monitoring.MetricType.NUM_METADATA_LOOKUP_FAILURES; +import static org.apache.phoenix.util.IndexUtil.isHintedGlobalIndex; /** * Validates FROM clause and builds a ColumnResolver for resolving column references @@ -1157,13 +1158,14 @@ public class FromCompiler { } private static class ProjectedTableColumnResolver extends MultiTableColumnResolver { - private final boolean isLocalIndex; + private final boolean isIndex; private final List<TableRef> theTableRefs; private final Map<ColumnRef, Integer> columnRefMap; private ProjectedTableColumnResolver(PTable projectedTable, PhoenixConnection conn, Map<String, UDFParseNode> udfParseNodes) throws SQLException { super(conn, 0, udfParseNodes, null); Preconditions.checkArgument(projectedTable.getType() == PTableType.PROJECTED); - this.isLocalIndex = projectedTable.getIndexType() == IndexType.LOCAL; + this.isIndex = projectedTable.getIndexType() == IndexType.LOCAL + || projectedTable.getIndexType() == IndexType.GLOBAL; this.columnRefMap = new HashMap<ColumnRef, Integer>(); long ts = Long.MAX_VALUE; for (int i = projectedTable.getBucketNum() == null ? 0 : 1; i < projectedTable.getColumns().size(); i++) { @@ -1201,9 +1203,11 @@ public class FromCompiler { try { colRef = super.resolveColumn(schemaName, tableName, colName); } catch (ColumnNotFoundException e) { - // This could be a ColumnRef for local index data column. - TableRef tableRef = isLocalIndex ? super.getTables().get(0) : super.resolveTable(schemaName, tableName); - if (tableRef.getTable().getIndexType() == IndexType.LOCAL) { + // This could be a ColumnRef for index data column. + TableRef tableRef = isIndex ? super.getTables().get(0) + : super.resolveTable(schemaName, tableName); + if (tableRef.getTable().getIndexType() == IndexType.LOCAL + || isHintedGlobalIndex(tableRef)) { try { TableRef parentTableRef = super.resolveTable( tableRef.getTable().getSchemaName().getString(), diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java index de49826..70345b5 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java @@ -20,6 +20,7 @@ package org.apache.phoenix.compile; import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT; import static org.apache.phoenix.schema.PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN; import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS; +import static org.apache.phoenix.util.IndexUtil.isHintedGlobalIndex; import java.sql.SQLException; import java.util.ArrayList; @@ -75,7 +76,7 @@ import org.apache.phoenix.parse.TableNodeVisitor; import org.apache.phoenix.parse.TableWildcardParseNode; import org.apache.phoenix.schema.ColumnNotFoundException; import org.apache.phoenix.schema.ColumnRef; -import org.apache.phoenix.schema.LocalIndexDataColumnRef; +import org.apache.phoenix.schema.IndexDataColumnRef; import org.apache.phoenix.schema.MetaDataEntityNotFoundException; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PNameFactory; @@ -1127,7 +1128,8 @@ public class JoinCompiler { if (columnRef.getTableRef().equals(tableRef) && (!retainPKColumns || !SchemaUtil.isPKColumn(columnRef.getColumn()))) { if (columnRef instanceof LocalIndexColumnRef) { - sourceColumns.add(new LocalIndexDataColumnRef(context, tableRef, IndexUtil.getIndexColumnName(columnRef.getColumn()))); + sourceColumns.add(new IndexDataColumnRef(context, tableRef, + IndexUtil.getIndexColumnName(columnRef.getColumn()))); } else { sourceColumns.add(columnRef); } @@ -1392,10 +1394,11 @@ public class JoinCompiler { try { columnRef = resolver.resolveColumn(node.getSchemaName(), node.getTableName(), node.getName()); } catch (ColumnNotFoundException e) { - // This could be a LocalIndexDataColumnRef. If so, the table name must have + // This could be an IndexDataColumnRef. If so, the table name must have // been appended by the IndexStatementRewriter, and we can convert it into. TableRef tableRef = resolver.resolveTable(node.getSchemaName(), node.getTableName()); - if (tableRef.getTable().getIndexType() == IndexType.LOCAL) { + if (tableRef.getTable().getIndexType() == IndexType.LOCAL + || isHintedGlobalIndex(tableRef)) { TableRef parentTableRef = FromCompiler.getResolver( NODE_FACTORY.namedTable(null, TableName.create(tableRef.getTable() .getSchemaName().getString(), tableRef.getTable() diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java index c13f383..f36f605 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java @@ -19,6 +19,7 @@ package org.apache.phoenix.compile; import static org.apache.phoenix.query.QueryServices.WILDCARD_QUERY_DYNAMIC_COLS_ATTRIB; import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_WILDCARD_QUERY_DYNAMIC_COLS_ATTRIB; +import static org.apache.phoenix.util.IndexUtil.isHintedGlobalIndex; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; @@ -69,9 +70,9 @@ import org.apache.phoenix.schema.ArgumentTypeMismatchException; import org.apache.phoenix.schema.ColumnFamilyNotFoundException; import org.apache.phoenix.schema.ColumnNotFoundException; import org.apache.phoenix.schema.ColumnRef; +import org.apache.phoenix.schema.IndexDataColumnRef; import org.apache.phoenix.schema.KeyValueSchema; import org.apache.phoenix.schema.KeyValueSchema.KeyValueSchemaBuilder; -import org.apache.phoenix.schema.LocalIndexDataColumnRef; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PColumnFamily; import org.apache.phoenix.schema.PDatum; @@ -230,9 +231,11 @@ public class ProjectionCompiler { indexColumn = index.getColumnForColumnName(indexColName); ref = new ColumnRef(tableRef, indexColumn.getPosition()); } catch (ColumnNotFoundException e) { - if (index.getIndexType() == IndexType.LOCAL) { + if (tableRef.getTable().getIndexType() == IndexType.LOCAL + || isHintedGlobalIndex(tableRef)) { try { - ref = new LocalIndexDataColumnRef(context, tableRef, indexColName); + context.setUncoveredIndex(true); + ref = new IndexDataColumnRef(context, tableRef, indexColName); indexColumn = ref.getColumn(); } catch (ColumnFamilyNotFoundException c) { throw e; @@ -303,9 +306,11 @@ public class ProjectionCompiler { ref = new ColumnRef(tableRef, indexColumn.getPosition()); indexColumnFamily = indexColumn.getFamilyName() == null ? null : indexColumn.getFamilyName().getString(); } catch (ColumnNotFoundException e) { - if (index.getIndexType() == IndexType.LOCAL) { + if (tableRef.getTable().getIndexType() == IndexType.LOCAL + || isHintedGlobalIndex(tableRef)) { try { - ref = new LocalIndexDataColumnRef(context, tableRef, indexColName); + context.setUncoveredIndex(true); + ref = new IndexDataColumnRef(context, tableRef, indexColName); indexColumn = ref.getColumn(); indexColumnFamily = indexColumn.getFamilyName() == null ? null @@ -702,7 +707,7 @@ public class ProjectionCompiler { PColumn col = expression.getColumn(); // hack'ish... For covered columns with local indexes we defer to the server. if (col instanceof ProjectedColumn && ((ProjectedColumn) col) - .getSourceColumnRef() instanceof LocalIndexDataColumnRef) { + .getSourceColumnRef() instanceof IndexDataColumnRef) { return null; } PTable table = context.getCurrentTable().getTable(); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java index 2abd546..d6cc20b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java @@ -87,6 +87,7 @@ public class StatementContext { private final OverAllQueryMetrics overAllQueryMetrics; private QueryLogger queryLogger; private boolean isClientSideUpsertSelect; + private boolean isUncoveredIndex; public StatementContext(PhoenixStatement statement) { this(statement, new Scan()); @@ -330,6 +331,14 @@ public class StatementContext { this.isClientSideUpsertSelect = isClientSideUpsertSelect; } + public boolean isUncoveredIndex() { + return isUncoveredIndex; + } + + public void setUncoveredIndex(boolean isUncoveredIndex) { + this.isUncoveredIndex = isUncoveredIndex; + } + /* * setRetryingPersistentCache can be used to override the USE_PERSISTENT_CACHE hint and disable the use of the * persistent cache for a specific cache ID. This can be used to retry queries that failed when using the persistent diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java index 2a67d8d..9174f50 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java @@ -18,6 +18,7 @@ package org.apache.phoenix.compile; import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_FAMILY; import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT; +import static org.apache.phoenix.util.IndexUtil.isHintedGlobalIndex; import java.sql.SQLException; import java.util.ArrayList; @@ -41,13 +42,12 @@ import org.apache.phoenix.parse.WildcardParseNode; import org.apache.phoenix.schema.ColumnFamilyNotFoundException; import org.apache.phoenix.schema.ColumnNotFoundException; import org.apache.phoenix.schema.ColumnRef; -import org.apache.phoenix.schema.LocalIndexDataColumnRef; +import org.apache.phoenix.schema.IndexDataColumnRef; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PNameFactory; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTable.EncodedCQCounter; -import org.apache.phoenix.schema.PTable.IndexType; import org.apache.phoenix.schema.PTableImpl; import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.ProjectedColumn; @@ -154,14 +154,18 @@ public class TupleProjectionCompiler { EncodedColumnsUtil.setColumns(column, table, context.getScan()); } } - // add LocalIndexDataColumnRef + // add IndexDataColumnRef position = projectedColumns.size(); - for (LocalIndexDataColumnRef sourceColumnRef : visitor.localIndexColumnRefSet) { + for (IndexDataColumnRef sourceColumnRef : visitor.indexColumnRefSet) { PColumn column = new ProjectedColumn(sourceColumnRef.getColumn().getName(), sourceColumnRef.getColumn().getFamilyName(), position++, sourceColumnRef.getColumn().isNullable(), sourceColumnRef, sourceColumnRef.getColumn().getColumnQualifierBytes()); projectedColumns.add(column); } + if (!visitor.indexColumnRefSet.isEmpty() + && tableRef.isHinted()) { + context.setUncoveredIndex(true); + } return PTableImpl.builderWithColumns(table, projectedColumns) .setType(PTableType.PROJECTED) .setBaseColumnCount(BASE_TABLE_BASE_COLUMN_COUNT) @@ -230,12 +234,12 @@ public class TupleProjectionCompiler { private static class ColumnRefVisitor extends StatelessTraverseAllParseNodeVisitor { private final StatementContext context; private final LinkedHashSet<ColumnRef> nonPkColumnRefSet; - private final LinkedHashSet<LocalIndexDataColumnRef> localIndexColumnRefSet; + private final LinkedHashSet<IndexDataColumnRef> indexColumnRefSet; private ColumnRefVisitor(StatementContext context) { this.context = context; this.nonPkColumnRefSet = new LinkedHashSet<ColumnRef>(); - this.localIndexColumnRefSet = new LinkedHashSet<LocalIndexDataColumnRef>(); + this.indexColumnRefSet = new LinkedHashSet<IndexDataColumnRef>(); } @Override @@ -247,9 +251,12 @@ public class TupleProjectionCompiler { nonPkColumnRefSet.add(resolveColumn); } } catch (ColumnNotFoundException e) { - if (context.getCurrentTable().getTable().getIndexType() == IndexType.LOCAL) { + if (context.getCurrentTable().getTable().getIndexType() == PTable.IndexType.LOCAL + || isHintedGlobalIndex(context.getCurrentTable())) { try { - localIndexColumnRefSet.add(new LocalIndexDataColumnRef(context, context.getCurrentTable(), node.getName())); + context.setUncoveredIndex(true); + indexColumnRefSet.add(new IndexDataColumnRef(context, + context.getCurrentTable(), node.getName())); } catch (ColumnFamilyNotFoundException c) { throw e; } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java index 9439a3c..1f6ab7f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java @@ -278,7 +278,9 @@ public class WhereCompiler { if (LiteralExpression.isBooleanFalseOrNull(whereClause)) { context.setScanRanges(ScanRanges.NOTHING); - } else if (context.getCurrentTable().getTable().getIndexType() == IndexType.LOCAL) { + } else if (context.getCurrentTable().getTable().getIndexType() == IndexType.LOCAL + || (context.getCurrentTable().getTable().getIndexType() == IndexType.GLOBAL + && context.isUncoveredIndex())) { if (whereClause != null && !ExpressionUtil.evaluatesToTrue(whereClause)) { // pass any extra where as scan attribute so it can be evaluated after all // columns from the main CF have been merged in @@ -291,11 +293,12 @@ public class WhereCompiler { } catch (IOException e) { throw new RuntimeException(e); } - scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX_FILTER, stream.toByteArray()); + scan.setAttribute(BaseScannerRegionObserver.INDEX_FILTER, stream.toByteArray()); // this is needed just for ExplainTable, since de-serializing an expression does not restore // its display properties, and that cannot be changed, due to backwards compatibility - scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX_FILTER_STR, Bytes.toBytes(whereClause.toString())); + scan.setAttribute(BaseScannerRegionObserver.INDEX_FILTER_STR, + Bytes.toBytes(whereClause.toString())); } } else if (whereClause != null && !ExpressionUtil.evaluatesToTrue(whereClause)) { Filter filter = null; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java index 9a63e88..64e2dae 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java @@ -86,6 +86,7 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver { public static final String GROUP_BY_LIMIT = "_GroupByLimit"; public static final String LOCAL_INDEX = "_LocalIndex"; public static final String LOCAL_INDEX_BUILD = "_LocalIndexBuild"; + public static final String UNCOVERED_GLOBAL_INDEX = "_UncoveredGlobalIndex"; public static final String INDEX_REBUILD_PAGING = "_IndexRebuildPaging"; // The number of index rows to be rebuild in one RPC call public static final String INDEX_REBUILD_PAGE_ROWS = "_IndexRebuildPageRows"; @@ -97,9 +98,15 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver { "_IndexRebuildDisableLoggingVerifyType"; public static final String INDEX_REBUILD_DISABLE_LOGGING_BEYOND_MAXLOOKBACK_AGE = "_IndexRebuildDisableLoggingBeyondMaxLookbackAge"; + @Deprecated public static final String LOCAL_INDEX_FILTER = "_LocalIndexFilter"; + @Deprecated public static final String LOCAL_INDEX_LIMIT = "_LocalIndexLimit"; + @Deprecated public static final String LOCAL_INDEX_FILTER_STR = "_LocalIndexFilterStr"; + public static final String INDEX_FILTER = "_IndexFilter"; + public static final String INDEX_LIMIT = "_IndexLimit"; + public static final String INDEX_FILTER_STR = "_IndexFilterStr"; /* * Attribute to denote that the index maintainer has been serialized using its proto-buf presentation. diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java index 00fd405..306f1fe 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java @@ -136,13 +136,8 @@ public class GroupedAggregateRegionObserver extends BaseScannerRegionObserver { .getEnvironment().getConfiguration(), em); RegionScanner innerScanner = s; - boolean useProto = false; - byte[] localIndexBytes = scan.getAttribute(LOCAL_INDEX_BUILD_PROTO); - useProto = localIndexBytes != null; - if (localIndexBytes == null) { - localIndexBytes = scan.getAttribute(LOCAL_INDEX_BUILD); - } - List<IndexMaintainer> indexMaintainers = localIndexBytes == null ? null : IndexMaintainer.deserialize(localIndexBytes, useProto); + List<IndexMaintainer> indexMaintainers = + IndexUtil.deSerializeIndexMaintainersFromScan(scan); TupleProjector tupleProjector = null; byte[][] viewConstants = null; ColumnReference[] dataColumns = IndexUtil.deserializeDataTableColumnsToJoin(scan); @@ -150,7 +145,8 @@ public class GroupedAggregateRegionObserver extends BaseScannerRegionObserver { final TupleProjector p = TupleProjector.deserializeProjectorFromScan(scan); final HashJoinInfo j = HashJoinInfo.deserializeHashJoinFromScan(scan); boolean useQualifierAsIndex = EncodedColumnsUtil.useQualifierAsIndex(EncodedColumnsUtil.getMinMaxQualifiersFromScan(scan)); - if (ScanUtil.isLocalIndex(scan) || (j == null && p != null)) { + if (ScanUtil.isLocalOrUncoveredGlobalIndex(scan) + || (j == null && p != null)) { if (dataColumns != null) { tupleProjector = IndexUtil.getTupleProjector(scan, dataColumns); viewConstants = IndexUtil.deserializeViewConstantsFromScan(scan); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java index d9aacd7..5494ca8 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java @@ -20,7 +20,6 @@ package org.apache.phoenix.execute; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.sql.ParameterMetaData; import java.sql.SQLException; import java.util.Collections; @@ -325,11 +324,16 @@ public abstract class BaseQueryPlan implements QueryPlan { ScanUtil.setTenantId(scan, tenantIdBytes); String customAnnotations = LogUtil.customAnnotationsToString(connection); - ScanUtil.setCustomAnnotations(scan, customAnnotations == null ? null - : customAnnotations.getBytes(StandardCharsets.UTF_8)); - // Set local index related scan attributes. - if (table.getIndexType() == IndexType.LOCAL) { - ScanUtil.setLocalIndex(scan); + ScanUtil.setCustomAnnotations(scan, + customAnnotations == null ? null : customAnnotations.getBytes()); + // Set index related scan attributes. + if (table.getType() == PTableType.INDEX) { + if (table.getIndexType() == IndexType.LOCAL) { + ScanUtil.setLocalIndex(scan); + } else if (context.isUncoveredIndex()) { + ScanUtil.setUncoveredGlobalIndex(scan); + } + Set<PColumn> dataColumns = context.getDataColumns(); // If any data columns to join back from data table are present then we set following attributes // 1. data columns to be projected and their key value schema. @@ -353,11 +357,12 @@ public abstract class BaseQueryPlan implements QueryPlan { KeyValueSchema schema = ProjectedColumnExpression.buildSchema(dataColumns); // Set key value schema of the data columns. serializeSchemaIntoScan(scan, schema); - - // Set index maintainer of the local index. - serializeIndexMaintainerIntoScan(scan, dataTable); - // Set view constants if exists. - serializeViewConstantsIntoScan(scan, dataTable); + if (table.getIndexType() == IndexType.LOCAL) { + // Set index maintainer of the local index. + serializeIndexMaintainerIntoScan(scan, dataTable); + // Set view constants if exists. + serializeViewConstantsIntoScan(scan, dataTable); + } } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index 59df237..26fee20 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -281,13 +281,14 @@ public abstract class BaseResultIterators extends ExplainTable implements Result } if (perScanLimit != null) { - if (scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_FILTER) == null) { + if (scan.getAttribute(BaseScannerRegionObserver.INDEX_FILTER) == null) { ScanUtil.andFilterAtEnd(scan, new PageFilter(perScanLimit)); } else { - // if we have a local index filter and a limit, handle the limit after the filter + // if we have an index filter and a limit, handle the limit after the filter // we cast the limit to a long even though it passed as an Integer so that // if we need extend this in the future the serialization is unchanged - scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX_LIMIT, Bytes.toBytes((long)perScanLimit)); + scan.setAttribute(BaseScannerRegionObserver.INDEX_LIMIT, + Bytes.toBytes((long) perScanLimit)); } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java index c3ab8f1..e0256c9 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java @@ -191,7 +191,11 @@ public abstract class ExplainTable { if (whereFilter != null) { whereFilterStr = whereFilter.toString(); } else { - byte[] expBytes = scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_FILTER_STR); + byte[] expBytes = scan.getAttribute(BaseScannerRegionObserver.INDEX_FILTER_STR); + if (expBytes == null) { + // For older clients + expBytes = scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_FILTER_STR); + } if (expBytes != null) { whereFilterStr = Bytes.toString(expBytes); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java index e258e2a..f371206 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java @@ -128,21 +128,15 @@ public class NonAggregateRegionScannerFactory extends RegionScannerFactory { PhoenixTransactionContext tx = null; ColumnReference[] dataColumns = IndexUtil.deserializeDataTableColumnsToJoin(scan); if (dataColumns != null) { - tupleProjector = IndexUtil.getTupleProjector(scan, dataColumns); - dataRegion = env.getRegion(); - boolean useProto = false; - byte[] localIndexBytes = scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_BUILD_PROTO); - useProto = localIndexBytes != null; - if (localIndexBytes == null) { - localIndexBytes = scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_BUILD); - } - int clientVersion = ScanUtil.getClientVersion(scan); - List<IndexMaintainer> indexMaintainers = - IndexMaintainer.deserialize(localIndexBytes, useProto); - indexMaintainer = indexMaintainers.get(0); - viewConstants = IndexUtil.deserializeViewConstantsFromScan(scan); - byte[] txState = scan.getAttribute(BaseScannerRegionObserver.TX_STATE); - tx = TransactionFactory.getTransactionContext(txState, clientVersion); + tupleProjector = IndexUtil.getTupleProjector(scan, dataColumns); + dataRegion = env.getRegion(); + int clientVersion = ScanUtil.getClientVersion(scan); + List<IndexMaintainer> indexMaintainers = + IndexUtil.deSerializeIndexMaintainersFromScan(scan); + indexMaintainer = indexMaintainers.get(0); + viewConstants = IndexUtil.deserializeViewConstantsFromScan(scan); + byte[] txState = scan.getAttribute(BaseScannerRegionObserver.TX_STATE); + tx = TransactionFactory.getTransactionContext(txState, clientVersion); } final TupleProjector p = TupleProjector.deserializeProjectorFromScan(scan); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/RegionScannerFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/RegionScannerFactory.java index 3426de8..8e2bde6 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/RegionScannerFactory.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/RegionScannerFactory.java @@ -135,9 +135,13 @@ public abstract class RegionScannerFactory { long extraLimit = -1; { - // for local indexes construct the row filter for uncovered columns if it exists - if (ScanUtil.isLocalIndex(scan)) { - byte[] expBytes = scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_FILTER); + // for indexes construct the row filter for uncovered columns if it exists + if (ScanUtil.isLocalOrUncoveredGlobalIndex(scan)) { + byte[] expBytes = scan.getAttribute(BaseScannerRegionObserver.INDEX_FILTER); + if (expBytes == null) { + // For older clients + expBytes = scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_FILTER); + } if (expBytes != null) { try { ByteArrayInputStream stream = new ByteArrayInputStream(expBytes); @@ -149,7 +153,11 @@ public abstract class RegionScannerFactory { throw new RuntimeException(io); } } - byte[] limitBytes = scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_LIMIT); + byte[] limitBytes = scan.getAttribute(BaseScannerRegionObserver.INDEX_LIMIT); + if (limitBytes == null) { + // For older clients + limitBytes = scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_LIMIT); + } if (limitBytes != null) { extraLimit = Bytes.toLong(limitBytes); } @@ -217,7 +225,8 @@ public abstract class RegionScannerFactory { if (result.size() == 0) { return next; } - if (ScanUtil.isLocalIndex(scan) && !ScanUtil.isAnalyzeTable(scan)) { + if ((ScanUtil.isLocalOrUncoveredGlobalIndex(scan)) + && !ScanUtil.isAnalyzeTable(scan)) { if(actualStartKey!=null) { next = scanTillScanStartRow(s, arrayKVRefs, arrayFuncRefs, result, null); @@ -229,8 +238,8 @@ public abstract class RegionScannerFactory { dataRegion will never be null in case of non-coprocessor call, therefore no need to refactor */ - IndexUtil.wrapResultUsingOffset(env, result, offset, dataColumns, - tupleProjector, dataRegion, indexMaintainer, viewConstants, ptr); + IndexUtil.wrapResultUsingOffset(env, result, scan, offset, dataColumns, + tupleProjector, dataRegion, indexMaintainer, viewConstants, ptr); if (extraWhere != null) { Tuple merged = useQualifierAsListIndex ? new PositionBasedResultTuple(result) : diff --git a/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java b/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java index ad21d9c..bd3282c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java @@ -334,6 +334,7 @@ public class QueryOptimizer { boolean isProjected = dataPlan.getContext().getResolver().getTables().get(0).getTable().getType() == PTableType.PROJECTED; // Check index state of now potentially updated index table to make sure it's active TableRef indexTableRef = resolver.getTables().get(0); + indexTableRef.setHinted(isHinted); PTable indexTable = indexTableRef.getTable(); PIndexState indexState = indexTable.getIndexState(); Map<TableRef, QueryPlan> dataPlans = Collections.singletonMap(indexTableRef, dataPlan); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/LocalIndexDataColumnRef.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/IndexDataColumnRef.java similarity index 65% rename from phoenix-core/src/main/java/org/apache/phoenix/schema/LocalIndexDataColumnRef.java rename to phoenix-core/src/main/java/org/apache/phoenix/schema/IndexDataColumnRef.java index 87f0999..3f45ebb 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/LocalIndexDataColumnRef.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/IndexDataColumnRef.java @@ -23,17 +23,23 @@ import java.util.Set; import org.apache.phoenix.compile.FromCompiler; import org.apache.phoenix.compile.StatementContext; import org.apache.phoenix.expression.ColumnExpression; +import org.apache.phoenix.expression.IsNullExpression; import org.apache.phoenix.expression.ProjectedColumnExpression; import org.apache.phoenix.parse.ParseNodeFactory; import org.apache.phoenix.parse.TableName; import org.apache.phoenix.util.IndexUtil; -public class LocalIndexDataColumnRef extends ColumnRef { +/** + * Even when a column is not covered by an index table for a given query, we may still want to + * use index in the query plan and fetch the missing columns from the data table rows on the + * server side. This class is used to keep track of such data columns. + */ +public class IndexDataColumnRef extends ColumnRef { final private int position; final private Set<PColumn> columns; private static final ParseNodeFactory FACTORY = new ParseNodeFactory(); - public LocalIndexDataColumnRef(StatementContext context, TableRef tRef, String indexColumnName) + public IndexDataColumnRef(StatementContext context, TableRef tRef, String indexColumnName) throws MetaDataEntityNotFoundException, SQLException { super(FromCompiler.getResolver( FACTORY.namedTable( @@ -48,15 +54,15 @@ public class LocalIndexDataColumnRef extends ColumnRef { columns = context.getDataColumns(); } - protected LocalIndexDataColumnRef(LocalIndexDataColumnRef localIndexDataColumnRef, long timestamp) { - super(localIndexDataColumnRef, timestamp); - this.position = localIndexDataColumnRef.position; - this.columns = localIndexDataColumnRef.columns; + protected IndexDataColumnRef(IndexDataColumnRef indexDataColumnRef, long timestamp) { + super(indexDataColumnRef, timestamp); + this.position = indexDataColumnRef.position; + this.columns = indexDataColumnRef.columns; } @Override public ColumnRef cloneAtTimestamp(long timestamp) { - return new LocalIndexDataColumnRef(this, timestamp); + return new IndexDataColumnRef(this, timestamp); } @Override @@ -64,4 +70,25 @@ public class LocalIndexDataColumnRef extends ColumnRef { String displayName = this.getTableRef().getColumnDisplayName(this, schemaNameCaseSensitive, colNameCaseSensitive); return new ProjectedColumnExpression(this.getColumn(), columns, position, displayName); } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + position; + result = prime * result + ((columns == null) ? 0 : columns.hashCode()); + return result; + } + + @Override + public boolean equals(Object o) { + if (!super.equals(o)) { + return false; + } + IndexDataColumnRef that = (IndexDataColumnRef) o; + if (position != that.position) { + return false; + } + return columns.equals(that.columns); + } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java index 5f426b0..1df722f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java @@ -38,6 +38,7 @@ public class TableRef { private final long lowerBoundTimeStamp; private final boolean hasDynamicCols; private final long currentTime; + private boolean hinted; private static TableRef createEmptyTableRef() { try { @@ -53,15 +54,18 @@ public class TableRef { } public TableRef(TableRef tableRef) { - this(tableRef.alias, tableRef.table, tableRef.upperBoundTimeStamp, tableRef.lowerBoundTimeStamp, tableRef.hasDynamicCols); + this(tableRef.alias, tableRef.table, tableRef.upperBoundTimeStamp, + tableRef.lowerBoundTimeStamp, tableRef.hasDynamicCols, tableRef.hinted); } public TableRef(TableRef tableRef, long timeStamp) { - this(tableRef.alias, tableRef.table, timeStamp, tableRef.lowerBoundTimeStamp, tableRef.hasDynamicCols); + this(tableRef.alias, tableRef.table, timeStamp, tableRef.lowerBoundTimeStamp, + tableRef.hasDynamicCols, tableRef.hinted); } public TableRef(TableRef tableRef, String alias) { - this(alias, tableRef.table, tableRef.upperBoundTimeStamp, tableRef.lowerBoundTimeStamp, tableRef.hasDynamicCols); + this(alias, tableRef.table, tableRef.upperBoundTimeStamp, tableRef.lowerBoundTimeStamp, + tableRef.hasDynamicCols, tableRef.hinted); } public TableRef(PTable table) { @@ -69,15 +73,20 @@ public class TableRef { } public TableRef(PTable table, long upperBoundTimeStamp, long lowerBoundTimeStamp) { - this(null, table, upperBoundTimeStamp, lowerBoundTimeStamp, false); + this(null, table, upperBoundTimeStamp, lowerBoundTimeStamp, false, false); } public TableRef(String alias, PTable table, long upperBoundTimeStamp, boolean hasDynamicCols) { - this(alias, table, upperBoundTimeStamp, 0, hasDynamicCols); + this(alias, table, upperBoundTimeStamp, 0, hasDynamicCols, false); + } + + public TableRef(String alias, PTable table, long upperBoundTimeStamp, long lowerBoundTimeStamp, + boolean hasDynamicCols) { + this(alias, table, upperBoundTimeStamp, lowerBoundTimeStamp, hasDynamicCols, false); } - public TableRef(String alias, PTable table, long upperBoundTimeStamp, long lowerBoundTimeStamp, - boolean hasDynamicCols) { + public TableRef(String alias, PTable table, long upperBoundTimeStamp, long lowerBoundTimeStamp, + boolean hasDynamicCols, boolean hinted) { this.alias = alias; this.table = table; // if UPDATE_CACHE_FREQUENCY is set, always let the server set timestamps @@ -85,6 +94,7 @@ public class TableRef { this.currentTime = this.upperBoundTimeStamp; this.lowerBoundTimeStamp = lowerBoundTimeStamp; this.hasDynamicCols = hasDynamicCols; + this.hinted = hinted; } public PTable getTable() { @@ -103,6 +113,14 @@ public class TableRef { return alias; } + public boolean isHinted() { + return hinted; + } + + public void setHinted(boolean hinted) { + this.hinted = hinted; + } + public String getColumnDisplayName(ColumnRef ref, boolean cfCaseSensitive, boolean cqCaseSensitive) { String cf = null; String cq = null; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java index 0bab683..c172f68 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java @@ -17,6 +17,9 @@ */ package org.apache.phoenix.util; +import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.LOCAL_INDEX_BUILD; +import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.LOCAL_INDEX_BUILD_PROTO; +import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.PHYSICAL_DATA_TABLE_NAME; import static org.apache.phoenix.coprocessor.MetaDataProtocol.PHOENIX_MAJOR_VERSION; import static org.apache.phoenix.coprocessor.MetaDataProtocol.PHOENIX_MINOR_VERSION; import static org.apache.phoenix.coprocessor.MetaDataProtocol.PHOENIX_PATCH_NUMBER; @@ -46,6 +49,8 @@ import org.apache.hadoop.hbase.PhoenixTagType; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; +import org.apache.phoenix.hbase.index.table.HTableFactory; +import org.apache.phoenix.index.PhoenixIndexCodec; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.thirdparty.com.google.common.cache.Cache; import org.apache.phoenix.thirdparty.com.google.common.cache.CacheBuilder; @@ -459,6 +464,25 @@ public class IndexUtil { } } + public static List<IndexMaintainer> deSerializeIndexMaintainersFromScan(Scan scan) { + boolean useProto = false; + byte[] indexBytes = scan.getAttribute(LOCAL_INDEX_BUILD_PROTO); + useProto = indexBytes != null; + if (indexBytes == null) { + indexBytes = scan.getAttribute(LOCAL_INDEX_BUILD); + } + if (indexBytes == null) { + indexBytes = scan.getAttribute(PhoenixIndexCodec.INDEX_PROTO_MD); + useProto = indexBytes != null; + } + if (indexBytes == null) { + indexBytes = scan.getAttribute(PhoenixIndexCodec.INDEX_MD); + } + List<IndexMaintainer> indexMaintainers = + indexBytes == null ? null : IndexMaintainer.deserialize(indexBytes, useProto); + return indexMaintainers; + } + public static byte[][] deserializeViewConstantsFromScan(Scan scan) { byte[] bytes = scan.getAttribute(BaseScannerRegionObserver.VIEW_CONSTANTS); if (bytes == null) return null; @@ -556,7 +580,7 @@ public class IndexUtil { } public static void wrapResultUsingOffset(final RegionCoprocessorEnvironment environment, - List<Cell> result, final int offset, ColumnReference[] dataColumns, + List<Cell> result, final Scan scan, final int offset, ColumnReference[] dataColumns, TupleProjector tupleProjector, Region dataRegion, IndexMaintainer indexMaintainer, byte[][] viewConstants, ImmutableBytesWritable ptr) throws IOException { if (tupleProjector != null) { @@ -576,18 +600,27 @@ public class IndexUtil { } } Result joinResult = null; - if (dataRegion != null) { - joinResult = dataRegion.get(get); - } else { - TableName dataTable = - TableName.valueOf(MetaDataUtil.getLocalIndexUserTableName( - environment.getRegion().getTableDesc().getNameAsString())); - HTableInterface table = null; - try { - table = environment.getTable(dataTable); + if (ScanUtil.isLocalIndex(scan)) { + if (dataRegion != null) { + joinResult = dataRegion.get(get); + } else { + TableName dataTable = + TableName.valueOf(MetaDataUtil.getLocalIndexUserTableName( + environment.getRegion().getTableDesc().getNameAsString())); + try (Table table = environment.getTable(dataTable)) { + joinResult = table.get(get); + } + } + } else if (ScanUtil.isUncoveredGlobalIndex(scan)) { + byte[] dataTableName = scan.getAttribute(PHYSICAL_DATA_TABLE_NAME); + + HTableFactory hTableFactory = ServerUtil.getDelegateHTableFactory(environment, + ServerUtil.ConnectionType.INDEX_WRITER_CONNECTION); + try (Table table = hTableFactory. + getTable(new ImmutableBytesPtr(dataTableName))) { joinResult = table.get(get); } finally { - if (table != null) table.close(); + hTableFactory.shutdown(); } } // at this point join result has data from the data table. We now need to take this result and @@ -981,6 +1014,13 @@ public class IndexUtil { } } + public static boolean isHintedGlobalIndex(final TableRef tableRef) { + PTable table = tableRef.getTable(); + return table.getType() == PTableType.INDEX + && table.getIndexType() == PTable.IndexType.GLOBAL + && tableRef.isHinted(); + } + /** * Updates the EMPTY cell value to VERIFIED for global index table rows. */ diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java index c1d3468..8901c50 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java @@ -138,9 +138,20 @@ public class ScanUtil { scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX, PDataType.TRUE_BYTES); } + public static void setUncoveredGlobalIndex(Scan scan) { + scan.setAttribute(BaseScannerRegionObserver.UNCOVERED_GLOBAL_INDEX, PDataType.TRUE_BYTES); + } + public static boolean isLocalIndex(Scan scan) { return scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX) != null; } + public static boolean isUncoveredGlobalIndex(Scan scan) { + return scan.getAttribute(BaseScannerRegionObserver.UNCOVERED_GLOBAL_INDEX) != null; + } + + public static boolean isLocalOrUncoveredGlobalIndex(Scan scan) { + return isLocalIndex(scan) || isUncoveredGlobalIndex(scan); + } public static boolean isNonAggregateScan(Scan scan) { return scan.getAttribute(BaseScannerRegionObserver.NON_AGGREGATE_QUERY) != null;