This is an automated email from the ASF dual-hosted git repository. shahrs87 pushed a commit to branch PHOENIX-6883-feature in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/PHOENIX-6883-feature by this push: new 39d7e94885 PHOENIX-7190 : Store LAST_DDL_TIMESTAMP of all ancestors in a View's PTable (#1807) 39d7e94885 is described below commit 39d7e94885cb5fc0e86e1d090d4398ade6a071fe Author: palash <palashc...@gmail.com> AuthorDate: Thu Feb 8 14:43:51 2024 -0800 PHOENIX-7190 : Store LAST_DDL_TIMESTAMP of all ancestors in a View's PTable (#1807) --- .../org/apache/phoenix/schema/DelegateTable.java | 5 + .../org/apache/phoenix/schema/MetaDataClient.java | 72 +++++++++-- .../java/org/apache/phoenix/schema/PTable.java | 4 + .../java/org/apache/phoenix/schema/PTableImpl.java | 15 ++- .../java/org/apache/phoenix/schema/PTableKey.java | 13 ++ .../phoenix/util/ValidateLastDDLTimestampUtil.java | 60 ++++----- .../org/apache/phoenix/end2end/ViewMetadataIT.java | 117 ++++++++++++++++++ .../java/org/apache/phoenix/rpc/UpdateCacheIT.java | 2 +- .../phoenix/cache/ServerMetadataCacheTest.java | 134 +++++++++++++++++++++ 9 files changed, 371 insertions(+), 51 deletions(-) diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/DelegateTable.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/DelegateTable.java index 1a5d83e728..453a7663d1 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/DelegateTable.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/DelegateTable.java @@ -413,6 +413,11 @@ public class DelegateTable implements PTable { return delegate.getIndexWhere(); } + @Override + public Map<PTableKey, Long> getAncestorLastDDLTimestampMap() { + return delegate.getAncestorLastDDLTimestampMap(); + } + @Override public Expression getIndexWhereExpression(PhoenixConnection connection) throws SQLException { diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 4878cb5cf4..0d63f089f5 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -716,8 +716,9 @@ public class MetaDataClient { // In this case, we update the parent table which may in turn pull // in indexes to add to this table. long resolvedTime = TransactionUtil.getResolvedTime(connection, result); - if (addColumnsAndIndexesFromAncestors(result, resolvedTimestamp, - true, false)) { + if (addColumnsIndexesAndLastDDLTimestampsFromAncestors(result, + resolvedTimestamp, true, false)) { + updateIndexesWithAncestorMap(result); connection.addTable(result.getTable(), resolvedTime); } else { // if we aren't adding the table, we still need to update the @@ -898,14 +899,14 @@ public class MetaDataClient { * @return true if the PTable contained by result was modified and false otherwise * @throws SQLException if the physical table cannot be found */ - private boolean addColumnsAndIndexesFromAncestors(MetaDataMutationResult result, Long resolvedTimestamp, - boolean alwaysAddAncestorColumnsAndIndexes, - boolean alwaysHitServerForAncestors) + private boolean addColumnsIndexesAndLastDDLTimestampsFromAncestors( + MetaDataMutationResult result, Long resolvedTimestamp, + boolean alwaysAddAncestorColumnsAndIndexes, + boolean alwaysHitServerForAncestors) throws SQLException { PTable table = result.getTable(); boolean hasIndexId = table.getViewIndexId() != null; - // only need to inherit columns and indexes for view indexes and views - if ((table.getType()==PTableType.INDEX && hasIndexId) + if (table.getType() == PTableType.INDEX || (table.getType() == PTableType.VIEW && table.getViewType() != ViewType.MAPPED)) { String tableName = null; try { @@ -940,8 +941,18 @@ public class MetaDataClient { if (!alwaysAddAncestorColumnsAndIndexes && !result.wasUpdated() && !parentResult.wasUpdated()) { return false; } - result.setTable(ViewUtil.addDerivedColumnsAndIndexesFromParent( - connection, table, parentTable)); + + // only need to inherit columns and indexes for view indexes and views + if (!table.getType().equals(PTableType.INDEX) || hasIndexId) { + PTable pTableWithDerivedColumnsAndIndexes + = ViewUtil.addDerivedColumnsAndIndexesFromParent(connection, + table, parentTable); + result.setTable(getPTableWithAncestorLastDDLTimestampMap( + pTableWithDerivedColumnsAndIndexes, parentTable)); + } else { + result.setTable(getPTableWithAncestorLastDDLTimestampMap( + table, parentTable)); + } return true; } catch (Throwable e) { TableMetricsManager.updateMetricsForSystemCatalogTableMethod(tableName, NUM_METADATA_LOOKUP_FAILURES, 1); @@ -951,6 +962,42 @@ public class MetaDataClient { return false; } + /** + * Update the indexes within this result's table with ancestor->last_ddl_timestamp map. + */ + private void updateIndexesWithAncestorMap(MetaDataMutationResult result) throws SQLException { + PTable table = result.getTable(); + if (table.getIndexes().isEmpty()) { + return; + } + List<PTable> newIndexes = new ArrayList<>(table.getIndexes().size()); + for (PTable index : table.getIndexes()) { + newIndexes.add(getPTableWithAncestorLastDDLTimestampMap(index, table)); + } + result.setTable(PTableImpl.builderWithColumns(table, PTableImpl.getColumnsToClone(table)) + .setIndexes(newIndexes).build()); + } + + /** + * Creates a new PTable object from the provided pTable and with the ancestorLastDDLTimestampMap + * Copy the map of the parent and add the last_ddl_timestamp of the parent in the map. + * @param pTable + * @param parentTable + */ + private PTable getPTableWithAncestorLastDDLTimestampMap(PTable pTable, PTable parentTable) + throws SQLException { + Map<PTableKey, Long> ancestorMap + = new HashMap<>(parentTable.getAncestorLastDDLTimestampMap()); + // this method can be called for an index and a view which inherited this index + // from its ancestors, skip adding the view as an ancestor of the index. + if (pTable.getParentName().equals(parentTable.getName())) { + ancestorMap.put(parentTable.getKey(), parentTable.getLastDDLTimestamp()); + } + return PTableImpl.builderWithColumns(pTable, PTableImpl.getColumnsToClone(pTable)) + .setAncestorLastDDLTimestampMap(ancestorMap) + .build(); + } + private void addFunctionArgMutation(String functionName, FunctionArgument arg, PreparedStatement argUpsert, int position) throws SQLException { argUpsert.setString(1, connection.getTenantId() == null ? null : connection.getTenantId().getString()); argUpsert.setString(2, functionName); @@ -5163,9 +5210,10 @@ public class MetaDataClient { private void addTableToCache(MetaDataMutationResult result, boolean alwaysHitServerForAncestors, long timestamp) throws SQLException { - addColumnsAndIndexesFromAncestors(result, null, false, alwaysHitServerForAncestors); - PTable table = result.getTable(); - connection.addTable(table, timestamp); + addColumnsIndexesAndLastDDLTimestampsFromAncestors(result, null, + false, alwaysHitServerForAncestors); + updateIndexesWithAncestorMap(result); + connection.addTable(result.getTable(), timestamp); } private void addFunctionToCache(MetaDataMutationResult result) throws SQLException { diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTable.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTable.java index 6fd89ad4bb..20f03fb621 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTable.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTable.java @@ -981,6 +981,10 @@ public interface PTable extends PMetaDataEntity { */ String getIndexWhere(); + /** + * @return the map of all ancestors to their LAST_DDL_TIMESTAMP + */ + Map<PTableKey, Long> getAncestorLastDDLTimestampMap(); /** * diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java index b5c2eee2e7..2885de65ea 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java @@ -218,6 +218,7 @@ public class PTableImpl implements PTable { private String indexWhere; private Expression indexWhereExpression; private Set<ColumnReference> indexWhereColumns; + private Map<PTableKey, Long> ancestorLastDDLTimestampMap; public static class Builder { private PTableKey key; @@ -284,6 +285,7 @@ public class PTableImpl implements PTable { private String externalSchemaId; private String streamingTopicName; private String indexWhere; + private Map<PTableKey, Long> ancestorLastDDLTimestampMap = new HashMap<>(); // Used to denote which properties a view has explicitly modified private BitSet viewModifiedPropSet = new BitSet(3); @@ -711,6 +713,10 @@ public class PTableImpl implements PTable { return this; } + public Builder setAncestorLastDDLTimestampMap(Map<PTableKey, Long> map) { + this.ancestorLastDDLTimestampMap = map; + return this; + } /** * Populate derivable attributes of the PTable * @return PTableImpl.Builder object @@ -1002,6 +1008,7 @@ public class PTableImpl implements PTable { this.externalSchemaId = builder.externalSchemaId; this.streamingTopicName = builder.streamingTopicName; this.indexWhere = builder.indexWhere; + this.ancestorLastDDLTimestampMap = builder.ancestorLastDDLTimestampMap; } // When cloning table, ignore the salt column as it will be added back in the constructor @@ -1082,7 +1089,8 @@ public class PTableImpl implements PTable { .setSchemaVersion(table.getSchemaVersion()) .setExternalSchemaId(table.getExternalSchemaId()) .setStreamingTopicName(table.getStreamingTopicName()) - .setIndexWhere(table.getIndexWhere()); + .setIndexWhere(table.getIndexWhere()) + .setAncestorLastDDLTimestampMap(table.getAncestorLastDDLTimestampMap()); } @Override @@ -2376,6 +2384,11 @@ public class PTableImpl implements PTable { return indexWhere; } + @Override + public Map<PTableKey, Long> getAncestorLastDDLTimestampMap() { + return ancestorLastDDLTimestampMap; + } + private void buildIndexWhereExpression(PhoenixConnection connection) throws SQLException { PhoenixPreparedStatement pstmt = diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableKey.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableKey.java index 874b62f3a8..a0204b8541 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableKey.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableKey.java @@ -20,10 +20,13 @@ package org.apache.phoenix.schema; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions; +import org.apache.phoenix.util.SchemaUtil; public class PTableKey { private final PName tenantId; private final String name; + private final String schemaName; + private final String tableName; public PTableKey(PName tenantId, String name) { Preconditions.checkNotNull(name); @@ -33,6 +36,8 @@ public class PTableKey { } else { this.name = name; } + this.schemaName = SchemaUtil.getSchemaNameFromFullName(this.name); + this.tableName = SchemaUtil.getTableNameFromFullName(this.name); } public PName getTenantId() { @@ -42,6 +47,14 @@ public class PTableKey { public String getName() { return name; } + + public String getSchemaName() { + return schemaName; + } + + public String getTableName() { + return tableName; + } @Override public String toString() { diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/util/ValidateLastDDLTimestampUtil.java b/phoenix-core-client/src/main/java/org/apache/phoenix/util/ValidateLastDDLTimestampUtil.java index 85acfe0753..01d58f2f11 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/util/ValidateLastDDLTimestampUtil.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/util/ValidateLastDDLTimestampUtil.java @@ -19,6 +19,7 @@ package org.apache.phoenix.util; import java.sql.SQLException; import java.util.List; +import java.util.Map; import java.util.concurrent.ThreadLocalRandom; import org.apache.hadoop.hbase.HConstants; @@ -33,8 +34,6 @@ import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTableKey; -import org.apache.phoenix.schema.PTableType; -import org.apache.phoenix.schema.TableNotFoundException; import org.apache.phoenix.schema.TableRef; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -135,7 +134,7 @@ public class ValidateLastDDLTimestampUtil { */ private static RegionServerEndpointProtos.ValidateLastDDLTimestampRequest getValidateDDLTimestampRequest(PhoenixConnection conn, List<TableRef> tableRefs, - boolean isWritePath) throws TableNotFoundException { + boolean isWritePath) { RegionServerEndpointProtos.ValidateLastDDLTimestampRequest.Builder requestBuilder = RegionServerEndpointProtos.ValidateLastDDLTimestampRequest.newBuilder(); @@ -143,45 +142,32 @@ public class ValidateLastDDLTimestampUtil { for (TableRef tableRef : tableRefs) { - //when querying an index, we need to validate its parent table - //in case the index was dropped - if (PTableType.INDEX.equals(tableRef.getTable().getType())) { + // validate all ancestors of this PTable if any + // index -> base table + // view -> parent view and its ancestors + // view index -> view and its ancestors + for (Map.Entry<PTableKey, Long> entry + : tableRef.getTable().getAncestorLastDDLTimestampMap().entrySet()) { innerBuilder = RegionServerEndpointProtos.LastDDLTimestampRequest.newBuilder(); - PTableKey key = new PTableKey(conn.getTenantId(), - tableRef.getTable().getParentName().getString()); - PTable parentTable = conn.getTable(key); - setLastDDLTimestampRequestParameters(innerBuilder, conn.getTenantId(), parentTable); + PTableKey ancestorKey = entry.getKey(); + setLastDDLTimestampRequestParameters(innerBuilder, ancestorKey, entry.getValue()); requestBuilder.addLastDDLTimestampRequests(innerBuilder); } - // add the tableRef to the request + // add the current table to the request + PTable ptable = tableRef.getTable(); innerBuilder = RegionServerEndpointProtos.LastDDLTimestampRequest.newBuilder(); - setLastDDLTimestampRequestParameters( - innerBuilder, conn.getTenantId(), tableRef.getTable()); + setLastDDLTimestampRequestParameters(innerBuilder, ptable.getKey(), + ptable.getLastDDLTimestamp()); requestBuilder.addLastDDLTimestampRequests(innerBuilder); - //when querying a view, we need to validate last ddl timestamps for all its ancestors - if (PTableType.VIEW.equals(tableRef.getTable().getType())) { - PTable pTable = tableRef.getTable(); - while (pTable.getParentName() != null) { - PTableKey key = new PTableKey(conn.getTenantId(), - pTable.getParentName().getString()); - PTable parentTable = conn.getTable(key); - innerBuilder = RegionServerEndpointProtos.LastDDLTimestampRequest.newBuilder(); - setLastDDLTimestampRequestParameters( - innerBuilder, conn.getTenantId(), parentTable); - requestBuilder.addLastDDLTimestampRequests(innerBuilder); - pTable = parentTable; - } - } - //on the write path, we need to validate all indexes of a table/view //in case index state was changed if (isWritePath) { for (PTable idxPTable : tableRef.getTable().getIndexes()) { innerBuilder = RegionServerEndpointProtos.LastDDLTimestampRequest.newBuilder(); - setLastDDLTimestampRequestParameters( - innerBuilder, conn.getTenantId(), idxPTable); + setLastDDLTimestampRequestParameters(innerBuilder, idxPTable.getKey(), + idxPTable.getLastDDLTimestamp()); requestBuilder.addLastDDLTimestampRequests(innerBuilder); } } @@ -194,16 +180,16 @@ public class ValidateLastDDLTimestampUtil { */ private static void setLastDDLTimestampRequestParameters( RegionServerEndpointProtos.LastDDLTimestampRequest.Builder builder, - PName tenantId, PTable pTable) { - byte[] tenantIDBytes = tenantId == null + PTableKey key, long lastDDLTimestamp) { + byte[] tenantIDBytes = key.getTenantId() == null ? HConstants.EMPTY_BYTE_ARRAY - : tenantId.getBytes(); - byte[] schemaBytes = pTable.getSchemaName() == null + : key.getTenantId().getBytes(); + byte[] schemaBytes = key.getSchemaName() == null ? HConstants.EMPTY_BYTE_ARRAY - : pTable.getSchemaName().getBytes(); + : key.getSchemaName().getBytes(); builder.setTenantId(ByteStringer.wrap(tenantIDBytes)); builder.setSchemaName(ByteStringer.wrap(schemaBytes)); - builder.setTableName(ByteStringer.wrap(pTable.getTableName().getBytes())); - builder.setLastDDLTimestamp(pTable.getLastDDLTimestamp()); + builder.setTableName(ByteStringer.wrap(key.getTableName().getBytes())); + builder.setLastDDLTimestamp(lastDDLTimestamp); } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewMetadataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewMetadataIT.java index 5c59eb3939..95cb40d8cd 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewMetadataIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewMetadataIT.java @@ -82,7 +82,10 @@ import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.ColumnAlreadyExistsException; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PName; +import org.apache.phoenix.schema.PNameImpl; import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTableImpl; +import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.TableAlreadyExistsException; import org.apache.phoenix.schema.TableNotFoundException; @@ -1371,6 +1374,120 @@ public class ViewMetadataIT extends SplitSystemCatalogIT { assertPKs(rs, new String[] {"K1", "K2", "K3", "K4"}); } + @Test + public void testAncestorLastDDLMapPopulatedInViewAndIndexHierarchy() throws SQLException { + String baseTable = SchemaUtil.getTableName(SCHEMA1, generateUniqueName()); + String view1 = SchemaUtil.getTableName(SCHEMA2, generateUniqueName()); + String view2 = SchemaUtil.getTableName(SCHEMA3, generateUniqueName()); + String view3 = SchemaUtil.getTableName(SCHEMA4, generateUniqueName()); + String view4 = SchemaUtil.getTableName(SCHEMA2, generateUniqueName()); + String index1 = generateUniqueName(); + String index2 = generateUniqueName(); + String tenant1 = TENANT1; + String tenant2 = TENANT2; + /* baseTable + / | \ \ + view1(tenant1) view3(tenant2) index1(global) view4(global) + / + view2(tenant1) + / + index2(tenant1) + */ + try (Connection conn = DriverManager.getConnection(getUrl())) { + String baseTableDDL = "CREATE TABLE " + baseTable + " (TENANT_ID VARCHAR NOT NULL, PK1 VARCHAR NOT NULL, V1 VARCHAR, V2 VARCHAR CONSTRAINT NAME_PK PRIMARY KEY(TENANT_ID, PK1)) MULTI_TENANT = true "; + conn.createStatement().execute(baseTableDDL); + String index1DDL = "CREATE INDEX " + index1 + " ON " + baseTable + "(V1)"; + conn.createStatement().execute(index1DDL); + + + try (Connection tenant1Conn = getTenantConnection(tenant1)) { + String view1DDL = "CREATE VIEW " + view1 + " AS SELECT * FROM " + baseTable; + tenant1Conn.createStatement().execute(view1DDL); + + String view2DDL = "CREATE VIEW " + view2 + " AS SELECT * FROM " + view1; + tenant1Conn.createStatement().execute(view2DDL); + + String index2DDL = "CREATE INDEX " + index2 + " ON " + view2 + "(V1)"; + tenant1Conn.createStatement().execute(index2DDL); + } + + try (Connection tenant2Conn = getTenantConnection(tenant2)) { + String view3DDL = "CREATE VIEW " + view3 + " AS SELECT * FROM " + baseTable; + tenant2Conn.createStatement().execute(view3DDL); + } + + String view4DDL = "CREATE VIEW " + view4 + " AS SELECT * FROM " + baseTable; + conn.createStatement().execute(view4DDL); + + //validate ancestor->last_ddl_timestamps maps + PTable basePTable = PhoenixRuntime.getTable(conn, baseTable); + Long baseTableLastDDLTimestamp = basePTable.getLastDDLTimestamp(); + PTableKey baseTableKey = new PTableKey(null, baseTable); + //base table map should be empty + Map<PTableKey,Long> map = basePTable.getAncestorLastDDLTimestampMap(); + assertEquals(0, map.size()); + + //global view + map = PhoenixRuntime.getTable(conn, view4).getAncestorLastDDLTimestampMap(); + assertEquals(1, map.size()); + assertEquals(baseTableLastDDLTimestamp, map.get(baseTableKey)); + + //global index in cache and in parent PTable + PTable index1PTable = PhoenixRuntime.getTable(conn, SchemaUtil.getTableName(SCHEMA1, index1)); + map = index1PTable.getAncestorLastDDLTimestampMap(); + assertEquals(baseTableLastDDLTimestamp, map.get(baseTableKey)); + assertEquals(1, basePTable.getIndexes().size()); + map = basePTable.getIndexes().get(0).getAncestorLastDDLTimestampMap(); + assertEquals(baseTableLastDDLTimestamp, map.get(baseTableKey)); + + //tenant2 view + try (Connection tenant2Conn = getTenantConnection(tenant2)) { + map = PhoenixRuntime.getTable(tenant2Conn, view3).getAncestorLastDDLTimestampMap(); + assertEquals(1, map.size()); + assertEquals(baseTableLastDDLTimestamp, map.get(baseTableKey)); + } + try (Connection tenant1Conn = getTenantConnection(tenant1)) { + //tenant1 view + PTable view1PTable = PhoenixRuntime.getTable(tenant1Conn, view1); + map = view1PTable.getAncestorLastDDLTimestampMap(); + assertEquals(1, map.size()); + assertEquals(baseTableLastDDLTimestamp, map.get(baseTableKey)); + //tenant1 child view + PTableKey view1Key = new PTableKey(view1PTable.getTenantId(), view1); + map = PhoenixRuntime.getTable(tenant1Conn, view2).getAncestorLastDDLTimestampMap(); + assertEquals(2, map.size()); + assertEquals(baseTableLastDDLTimestamp, map.get(baseTableKey)); + assertEquals(view1PTable.getLastDDLTimestamp(), map.get(view1Key)); + //tenant1 child view index in cache and in child view PTable + PTable view2PTable = PhoenixRuntime.getTable(tenant1Conn, view2); + PTableKey view2Key = new PTableKey(view2PTable.getTenantId(), view2); + PTable index2PTable = PhoenixRuntime.getTable(tenant1Conn, SchemaUtil.getTableName(SCHEMA3, index2)); + map = index2PTable.getAncestorLastDDLTimestampMap(); + assertEquals(baseTableLastDDLTimestamp, map.get(baseTableKey)); + assertEquals(view2PTable.getLastDDLTimestamp(), map.get(view2Key)); + assertEquals(2, view2PTable.getIndexes().size()); + for (PTable index : view2PTable.getIndexes()) { + // inherited index + if (index.getTableName().getString().equals(index1)) { + map = index.getAncestorLastDDLTimestampMap(); + assertEquals(baseTableLastDDLTimestamp, map.get(baseTableKey)); + } else { + // view index + map = index.getAncestorLastDDLTimestampMap(); + assertEquals(baseTableLastDDLTimestamp, map.get(baseTableKey)); + assertEquals(view2PTable.getLastDDLTimestamp(), map.get(view2Key)); + } + } + } + } + } + + private Connection getTenantConnection(String tenantId) throws SQLException { + Properties tenantProps = new Properties(); + tenantProps.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId); + return DriverManager.getConnection(getUrl(), tenantProps); + } + private void assertPKs(ResultSet rs, String[] expectedPKs) throws SQLException { List<String> pkCols = newArrayListWithExpectedSize(expectedPKs.length); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java index 93cd74947a..7be7d1c715 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java @@ -231,7 +231,7 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT { // Even the indexes should now have the modified value of UPDATE_CACHE_FREQUENCY // Note that when we query the base table, during query plan generation, we make 2 getTable // requests (to retrieve the base table) for each index of the base table - helpTestUpdateCache(fullTableName, new int[] {1, 18}, false); + helpTestUpdateCache(fullTableName, new int[] {1, 21}, false); helpTestUpdateCache(INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + localIndex, new int[] {3}, true); helpTestUpdateCache(INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + globalIndex, diff --git a/phoenix-core/src/test/java/org/apache/phoenix/cache/ServerMetadataCacheTest.java b/phoenix-core/src/test/java/org/apache/phoenix/cache/ServerMetadataCacheTest.java index f857b670c8..ec70879b17 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/cache/ServerMetadataCacheTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/cache/ServerMetadataCacheTest.java @@ -31,6 +31,7 @@ import org.apache.phoenix.schema.ColumnNotFoundException; import org.apache.phoenix.schema.ConnectionProperty; import org.apache.phoenix.schema.PIndexState; import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.schema.TableNotFoundException; import org.apache.phoenix.schema.types.PVarchar; import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; @@ -1458,6 +1459,139 @@ public class ServerMetadataCacheTest extends ParallelStatsDisabledIT { } } + /** + * Test that a query on the column of a view which was previously dropped + * throws a ColumnNotFoundException. Use the same client to drop the column. + */ + @Test + public void testDroppedTableColumnNotVisibleToViewUsingSameClient() throws Exception { + testDroppedTableColumnNotVisibleToView(true); + } + + /** + * Test that a query on the column of a view which was previously dropped + * throws a ColumnNotFoundException. Use a different client to drop the column. + */ + @Test + public void testDroppedTableColumnNotVisibleToViewUsingDifferentClients() throws Exception { + testDroppedTableColumnNotVisibleToView(false); + } + + public void testDroppedTableColumnNotVisibleToView(boolean useSameClient) throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + String url1 = QueryUtil.getConnectionUrl(props, config, "client1"); + String url2 = QueryUtil.getConnectionUrl(props, config, "client2"); + String tableName = generateUniqueName(); + String viewName1 = generateUniqueName(); + String viewName2 = generateUniqueName(); + ConnectionQueryServices cqs1 = driver.getConnectionQueryServices(url1, props); + ConnectionQueryServices cqs2 = driver.getConnectionQueryServices(url2, props); + try (Connection conn = cqs1.connect(url1, props); + Connection conn2 = useSameClient ? conn : cqs2.connect(url2, props)) { + createTable(conn, tableName, NEVER); + createView(conn, tableName, viewName1); + createView(conn, viewName1, viewName2); + query(conn2, viewName2); + + alterTableDropColumn(conn, tableName, "v2"); + query(conn2, tableName); + + conn2.createStatement().execute("SELECT v2 FROM " + viewName2); + fail("Column dropped from base table should not be visible to view."); + } catch (ColumnNotFoundException expected) { + } + } + + /** + * Test that ancestor->last_ddl_timestamp is populated in a new client. + * @throws Exception + */ + @Test + public void testAncestorLastDDLMapPopulatedInDifferentClient() throws Exception { + String SCHEMA1 = generateUniqueName(); + String SCHEMA2 = generateUniqueName(); + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + String baseTable = SchemaUtil.getTableName(SCHEMA1, generateUniqueName()); + String index = generateUniqueName(); + String view = SchemaUtil.getTableName(SCHEMA2, generateUniqueName()); + String viewIndex = generateUniqueName(); + String baseTable2 = SchemaUtil.getTableName(SCHEMA1, generateUniqueName()); + String index2 = generateUniqueName(); + String view2 = SchemaUtil.getTableName(SCHEMA2, generateUniqueName()); + String viewIndex2 = generateUniqueName(); + String url1 = QueryUtil.getConnectionUrl(props, config, "client1"); + String url2 = QueryUtil.getConnectionUrl(props, config, "client2"); + ConnectionQueryServices cqs1 = driver.getConnectionQueryServices(url1, props); + ConnectionQueryServices cqs2 = driver.getConnectionQueryServices(url2, props); + try (Connection conn = cqs1.connect(url1, props); + Connection conn2 = cqs2.connect(url2, props)) { + //client-1 creates tables, views, indexes and view indexes + createTable(conn, baseTable, NEVER); + createView(conn, baseTable, view); + createIndex(conn, baseTable, index, "v2"); + createIndex(conn, view, viewIndex, "v1"); + createTable(conn, baseTable2, NEVER); + createView(conn, baseTable2, view2); + createIndex(conn, baseTable2, index2, "v2"); + createIndex(conn, view2, viewIndex2, "v1"); + + //client-2 queries the view + query(conn2, view); + + PTable basePTable = PhoenixRuntime.getTable(conn2, baseTable); + PTable viewPTable = PhoenixRuntime.getTable(conn2, view); + PTable viewIndexPTable = PhoenixRuntime.getTable(conn2, SchemaUtil.getTableName(SCHEMA2, viewIndex)); + PTable indexPTable = PhoenixRuntime.getTable(conn2, SchemaUtil.getTableName(SCHEMA1, index)); + + //verify view has base table in ancestor map + Map<PTableKey,Long> map = viewPTable.getAncestorLastDDLTimestampMap(); + assertEquals(basePTable.getLastDDLTimestamp(), map.get(basePTable.getKey())); + + //verify view index has base table and view in ancestor map + map = viewIndexPTable.getAncestorLastDDLTimestampMap(); + assertEquals(2, map.size()); + assertEquals(basePTable.getLastDDLTimestamp(), map.get(basePTable.getKey())); + assertEquals(viewPTable.getLastDDLTimestamp(), map.get(viewPTable.getKey())); + + //verify index has only base table in ancestor map + map = indexPTable.getAncestorLastDDLTimestampMap(); + assertEquals(1, map.size()); + assertEquals(basePTable.getLastDDLTimestamp(), map.get(basePTable.getKey())); + + //also verify index PTable within base table has the map + assertEquals(1, basePTable.getIndexes().size()); + map = basePTable.getIndexes().get(0).getAncestorLastDDLTimestampMap(); + assertEquals(1, map.size()); + assertEquals(basePTable.getLastDDLTimestamp(), map.get(basePTable.getKey())); + + //verify client-2 sees maps directly through PhoenixRuntime, no query on baseTable2 or view2 + PTable basePTable2 = PhoenixRuntime.getTable(conn2, baseTable2); + map = basePTable2.getAncestorLastDDLTimestampMap(); + assertEquals(0, map.size()); + assertEquals(1, basePTable2.getIndexes().size()); + map = basePTable2.getIndexes().get(0).getAncestorLastDDLTimestampMap(); + assertEquals(basePTable2.getLastDDLTimestamp(), map.get(basePTable2.getKey())); + + PTable viewPTable2 = PhoenixRuntime.getTable(conn2, view2); + map = viewPTable2.getAncestorLastDDLTimestampMap(); + assertEquals(basePTable2.getLastDDLTimestamp(), map.get(basePTable2.getKey())); + assertEquals(2, viewPTable2.getIndexes().size()); + for (PTable indexT : viewPTable2.getIndexes()) { + // inherited index + if (indexT.getTableName().getString().equals(index2)) { + map = indexT.getAncestorLastDDLTimestampMap(); + assertEquals(basePTable2.getLastDDLTimestamp(), map.get(basePTable2.getKey())); + } else { + // view index + map = indexT.getAncestorLastDDLTimestampMap(); + assertEquals(basePTable2.getLastDDLTimestamp(), map.get(basePTable2.getKey())); + assertEquals(viewPTable2.getLastDDLTimestamp(), map.get(viewPTable2.getKey())); + } + } + } + } + + //Helper methods private long getLastDDLTimestamp(String tableName) throws SQLException { Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);