This is an automated email from the ASF dual-hosted git repository.
shahrs87 pushed a commit to branch PHOENIX-6883-feature
in repository https://gitbox.apache.org/repos/asf/phoenix.git
commit 124d7dbbf58452f726acc453770369b5932ae618
Merge: 4d42c915e1 d8f6e6ea53
Author: Rushabh Shah <rushabh.s...@rushabh-ltmflld.internal.salesforce.com>
AuthorDate: Mon Apr 8 12:00:09 2024 -0700
Merge branch 'master' into PHOENIX-6883-feature
.asf.yaml | 23 +-
.../phoenix/compile/ExplainPlanAttributes.java | 21 +-
.../org/apache/phoenix/compile/QueryCompiler.java | 17 +-
.../phoenix/compile/ServerBuildIndexCompiler.java | 1 +
.../ServerBuildTransformingTableCompiler.java | 1 +
.../BaseScannerRegionObserverConstants.java | 2 +-
.../coprocessorclient/MetaDataProtocol.java | 2 +-
.../apache/phoenix/exception/SQLExceptionCode.java | 1 +
.../phoenix/iterate/BaseResultIterators.java | 57 +++-
.../iterate/DefaultParallelScanGrouper.java | 11 +
.../phoenix/iterate/ParallelScanGrouper.java | 16 +
.../phoenix/iterate/ScanningResultIterator.java | 5 +-
.../phoenix/jdbc/PhoenixDatabaseMetaData.java | 3 +
.../apache/phoenix/optimize/QueryOptimizer.java | 5 +-
.../phoenix/query/ConnectionQueryServices.java | 15 +
.../phoenix/query/ConnectionQueryServicesImpl.java | 103 +++++--
.../query/ConnectionlessQueryServicesImpl.java | 16 +-
.../query/DelegateConnectionQueryServices.java | 9 +
.../org/apache/phoenix/query/QueryConstants.java | 2 +
.../org/apache/phoenix/query/QueryServices.java | 7 +
.../org/apache/phoenix/schema/DelegateTable.java | 6 +
.../org/apache/phoenix/schema/MetaDataClient.java | 71 ++++-
.../java/org/apache/phoenix/schema/PTable.java | 5 +
.../java/org/apache/phoenix/schema/PTableImpl.java | 28 +-
.../org/apache/phoenix/schema/TableProperty.java | 24 ++
.../phoenix/schema/transform/TransformClient.java | 1 +
.../java/org/apache/phoenix/util/MetaDataUtil.java | 14 +
.../java/org/apache/phoenix/util/ScanUtil.java | 15 +
phoenix-core-client/src/main/protobuf/PTable.proto | 1 +
.../coprocessor/BaseScannerRegionObserver.java | 50 ++-
.../coprocessor/GlobalIndexRegionScanner.java | 4 +-
.../phoenix/coprocessor/MetaDataEndpointImpl.java | 121 +++++++-
.../UngroupedAggregateRegionObserver.java | 4 +-
.../iterate/MapReduceParallelScanGrouper.java | 48 ++-
.../PhoenixServerBuildIndexInputFormat.java | 1 +
.../mapreduce/index/IndexScrutinyMapper.java | 3 +-
.../phoenix/mapreduce/index/IndexScrutinyTool.java | 31 +-
.../mapreduce/util/PhoenixConfigurationUtil.java | 14 +
.../org/apache/phoenix/end2end/AlterTableIT.java | 106 ++++++-
.../end2end/BaseTenantSpecificViewIndexIT.java | 4 +-
.../org/apache/phoenix/end2end/CreateTableIT.java | 78 +++++
.../end2end/ExplainPlanWithStatsDisabledIT.java | 340 +++++++++++++++++++++
.../end2end/IndexRepairRegionScannerIT.java | 111 ++++---
.../phoenix/end2end/IndexScrutinyToolBaseIT.java | 7 +-
.../end2end/IndexScrutinyWithMaxLookbackIT.java | 120 ++++++--
.../end2end/IndexToolForNonTxGlobalIndexIT.java | 61 ++++
.../phoenix/end2end/MaxLookbackExtendedIT.java | 75 ++++-
.../org/apache/phoenix/end2end/MaxLookbackIT.java | 251 +++++++++++++++
.../end2end/RowValueConstructorOffsetIT.java | 26 ++
.../org/apache/phoenix/end2end/TableTTLIT.java | 34 ++-
.../phoenix/end2end/TenantSpecificTablesDDLIT.java | 57 ++++
.../end2end/TenantSpecificViewIndexSaltedIT.java | 33 +-
.../it/java/org/apache/phoenix/end2end/ViewIT.java | 85 ++++++
.../org/apache/phoenix/end2end/ViewMetadataIT.java | 34 +++
.../phoenix/end2end/index/IndexMetadataIT.java | 24 ++
.../phoenix/end2end/index/PartialIndexIT.java | 47 +++
.../index/UncoveredGlobalIndexRegionScannerIT.java | 43 +++
.../phoenix/end2end/salted/SaltedTableIT.java | 32 ++
.../phoenix/end2end/transform/TransformToolIT.java | 144 ++++++++-
.../phoenix/iterate/PhoenixQueryTimeoutIT.java | 55 ++++
.../monitoring/PhoenixTableLevelMetricsIT.java | 14 +
.../it/resources/compatible_client_versions.json | 8 +-
.../src/it/resources/log4j2-test.properties | 45 ---
.../TestingMapReduceParallelScanGrouper.java | 10 +
.../java/org/apache/phoenix/query/BaseTest.java | 44 +++
.../query/ConnectionQueryServicesImplTest.java | 46 ++-
.../org/apache/phoenix/query/QueryPlanTest.java | 12 +-
.../java/org/apache/phoenix/util/TestUtil.java | 4 +-
.../src/test/resources/log4j2-test.properties | 8 +-
pom.xml | 42 ++-
70 files changed, 2471 insertions(+), 287 deletions(-)
diff --cc
phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 40da589976,0e78202c43..d4e2fd7552
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@@ -3365,9 -3321,10 +3384,10 @@@ public class MetaDataClient
.setStreamingTopicName(streamingTopicName)
.setIndexWhere(statement.getWhereClause() == null ?
null
: statement.getWhereClause().toString())
+ .setMaxLookbackAge(maxLookbackAge)
.build();
result = new MetaDataMutationResult(code,
result.getMutationTime(), table, true);
- addTableToCache(result);
+ addTableToCache(result, false);
return table;
} catch (Throwable e) {
TableMetricsManager.updateMetricsForSystemCatalogTableMethod(tableNameNode.toString(),
diff --cc
phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java
index 2885de65ea,128bca0d7f..2623d7709e
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java
@@@ -218,7 -219,7 +219,8 @@@ public class PTableImpl implements PTab
private String indexWhere;
private Expression indexWhereExpression;
private Set<ColumnReference> indexWhereColumns;
+ private Long maxLookbackAge;
+ private Map<PTableKey, Long> ancestorLastDDLTimestampMap;
public static class Builder {
private PTableKey key;
@@@ -285,8 -286,8 +287,8 @@@
private String externalSchemaId;
private String streamingTopicName;
private String indexWhere;
+ private Long maxLookbackAge;
-
+ private Map<PTableKey, Long> ancestorLastDDLTimestampMap = new
HashMap<>();
-
// Used to denote which properties a view has explicitly modified
private BitSet viewModifiedPropSet = new BitSet(3);
// Optionally set columns for the builder, but not for the actual
PTable
@@@ -713,10 -714,14 +715,19 @@@
return this;
}
+ public Builder setMaxLookbackAge(Long maxLookbackAge) {
+ if (maxLookbackAge != null) {
+ propertyValues.put(MAX_LOOKBACK_AGE,
String.valueOf(maxLookbackAge));
+ }
+ this.maxLookbackAge = maxLookbackAge;
+ return this;
+ }
+
+ public Builder setAncestorLastDDLTimestampMap(Map<PTableKey, Long>
map) {
+ this.ancestorLastDDLTimestampMap = map;
+ return this;
+ }
++
/**
* Populate derivable attributes of the PTable
* @return PTableImpl.Builder object
@@@ -1008,7 -1013,7 +1019,8 @@@
this.externalSchemaId = builder.externalSchemaId;
this.streamingTopicName = builder.streamingTopicName;
this.indexWhere = builder.indexWhere;
+ this.maxLookbackAge = builder.maxLookbackAge;
+ this.ancestorLastDDLTimestampMap =
builder.ancestorLastDDLTimestampMap;
}
// When cloning table, ignore the salt column as it will be added back in
the constructor
@@@ -1090,7 -1095,7 +1102,8 @@@
.setExternalSchemaId(table.getExternalSchemaId())
.setStreamingTopicName(table.getStreamingTopicName())
.setIndexWhere(table.getIndexWhere())
- .setMaxLookbackAge(table.getMaxLookbackAge());
++ .setMaxLookbackAge(table.getMaxLookbackAge())
+
.setAncestorLastDDLTimestampMap(table.getAncestorLastDDLTimestampMap());
}
@Override
@@@ -2384,11 -2397,11 +2405,16 @@@
return indexWhere;
}
+ @Override
+ public Long getMaxLookbackAge() {
+ return maxLookbackAge;
+ }
+
+ @Override
+ public Map<PTableKey, Long> getAncestorLastDDLTimestampMap() {
+ return ancestorLastDDLTimestampMap;
+ }
+
private void buildIndexWhereExpression(PhoenixConnection connection)
throws SQLException {
PhoenixPreparedStatement
pstmt =
diff --cc
phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index 6eb7577e95,bdba476a90..d9d200a4d8
---
a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++
b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@@ -83,7 -84,9 +84,10 @@@ import static org.apache.phoenix.jdbc.P
import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_DATA_TYPE_BYTES;
import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE_BYTES;
+import static
org.apache.phoenix.query.QueryServices.SKIP_SYSTEM_TABLES_EXISTENCE_CHECK;
+ import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MAX_LOOKBACK_AGE_BYTES;
+ import static org.apache.phoenix.schema.PTable.LinkType.PHYSICAL_TABLE;
+ import static
org.apache.phoenix.schema.PTable.LinkType.VIEW_INDEX_PARENT_TABLE;
import static org.apache.phoenix.schema.PTableImpl.getColumnsToClone;
import static org.apache.phoenix.schema.PTableType.INDEX;
import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB;
@@@ -3459,19 -3513,32 +3550,45 @@@ TABLE_FAMILY_BYTES, TABLE_SEQ_NUM_BYTES
}
}
+ private void
invalidateServerMetadataCache(List<InvalidateServerMetadataCacheRequest>
requests)
+ throws Throwable {
+ Properties properties = new Properties();
+ // Skip checking of system table existence since the system tables
should have created
+ // by now.
+ properties.setProperty(SKIP_SYSTEM_TABLES_EXISTENCE_CHECK, "true");
+ try (PhoenixConnection connection =
QueryUtil.getConnectionOnServer(properties,
-
env.getConfiguration()).unwrap(PhoenixConnection.class)) {
++ env.getConfiguration()).unwrap(PhoenixConnection.class)) {
+ ConnectionQueryServices queryServices =
connection.getQueryServices();
+ queryServices.invalidateServerMetadataCache(requests);
+ }
+ }
+
+ private boolean hasInheritableTablePropertyChanged(PTable newTable,
PTable oldTable) {
+ return ! Objects.equals(newTable.getMaxLookbackAge(),
oldTable.getMaxLookbackAge());
+ }
+
+ private void invalidateAllChildTablesAndIndexes(PTable table,
List<PTable> childViews) {
+ List<ImmutableBytesPtr> invalidateList = new
ArrayList<ImmutableBytesPtr>();
+ if (table.getIndexes() != null) {
+ for(PTable index: table.getIndexes()) {
+ invalidateList.add(new
ImmutableBytesPtr(SchemaUtil.getTableKey(index)));
+ }
+ }
+ for(PTable childView: childViews) {
+ invalidateList.add(new
ImmutableBytesPtr(SchemaUtil.getTableKey(childView)));
+ if (childView.getIndexes() != null) {
+ for(PTable viewIndex: childView.getIndexes()) {
+ invalidateList.add(new
ImmutableBytesPtr(SchemaUtil.getTableKey(viewIndex)));
+ }
+ }
+ }
+ Cache<ImmutableBytesPtr, PMetaDataEntity> metaDataCache =
+ GlobalCache.getInstance(this.env).getMetaDataCache();
+ for(ImmutableBytesPtr invalidateKey: invalidateList) {
+ metaDataCache.invalidate(invalidateKey);
+ }
+ }
+
/**
* Removes the table from the server side cache
*/
diff --cc
phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewMetadataIT.java
index 47b0818953,cb07031ac5..c32a8e2d4f
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewMetadataIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewMetadataIT.java
@@@ -1374,120 -1375,36 +1378,150 @@@ public class ViewMetadataIT extends Spl
assertPKs(rs, new String[] {"K1", "K2", "K3", "K4"});
}
+ @Test
+ public void testAncestorLastDDLMapPopulatedInViewAndIndexHierarchy()
throws SQLException {
+ String baseTable = SchemaUtil.getTableName(SCHEMA1,
generateUniqueName());
+ String view1 = SchemaUtil.getTableName(SCHEMA2, generateUniqueName());
+ String view2 = SchemaUtil.getTableName(SCHEMA3, generateUniqueName());
+ String view3 = SchemaUtil.getTableName(SCHEMA4, generateUniqueName());
+ String view4 = SchemaUtil.getTableName(SCHEMA2, generateUniqueName());
+ String index1 = generateUniqueName();
+ String index2 = generateUniqueName();
+ String tenant1 = TENANT1;
+ String tenant2 = TENANT2;
+ /* baseTable
+ / | \
\
+ view1(tenant1) view3(tenant2) index1(global)
view4(global)
+ /
+ view2(tenant1)
+ /
+ index2(tenant1)
+ */
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ String baseTableDDL = "CREATE TABLE " + baseTable + " (TENANT_ID
VARCHAR NOT NULL, PK1 VARCHAR NOT NULL, V1 VARCHAR, V2 VARCHAR CONSTRAINT
NAME_PK PRIMARY KEY(TENANT_ID, PK1)) MULTI_TENANT = true ";
+ conn.createStatement().execute(baseTableDDL);
+ String index1DDL = "CREATE INDEX " + index1 + " ON " + baseTable
+ "(V1)";
+ conn.createStatement().execute(index1DDL);
+
+
+ try (Connection tenant1Conn = getTenantConnection(tenant1)) {
+ String view1DDL = "CREATE VIEW " + view1 + " AS SELECT * FROM
" + baseTable;
+ tenant1Conn.createStatement().execute(view1DDL);
+
+ String view2DDL = "CREATE VIEW " + view2 + " AS SELECT * FROM
" + view1;
+ tenant1Conn.createStatement().execute(view2DDL);
+
+ String index2DDL = "CREATE INDEX " + index2 + " ON " + view2
+ "(V1)";
+ tenant1Conn.createStatement().execute(index2DDL);
+ }
+
+ try (Connection tenant2Conn = getTenantConnection(tenant2)) {
+ String view3DDL = "CREATE VIEW " + view3 + " AS SELECT * FROM
" + baseTable;
+ tenant2Conn.createStatement().execute(view3DDL);
+ }
+
+ String view4DDL = "CREATE VIEW " + view4 + " AS SELECT * FROM " +
baseTable;
+ conn.createStatement().execute(view4DDL);
+
+ //validate ancestor->last_ddl_timestamps maps
+ PTable basePTable = PhoenixRuntime.getTable(conn, baseTable);
+ Long baseTableLastDDLTimestamp = basePTable.getLastDDLTimestamp();
+ PTableKey baseTableKey = new PTableKey(null, baseTable);
+ //base table map should be empty
+ Map<PTableKey,Long> map =
basePTable.getAncestorLastDDLTimestampMap();
+ assertEquals(0, map.size());
+
+ //global view
+ map = PhoenixRuntime.getTable(conn,
view4).getAncestorLastDDLTimestampMap();
+ assertEquals(1, map.size());
+ assertEquals(baseTableLastDDLTimestamp, map.get(baseTableKey));
+
+ //global index in cache and in parent PTable
+ PTable index1PTable = PhoenixRuntime.getTable(conn,
SchemaUtil.getTableName(SCHEMA1, index1));
+ map = index1PTable.getAncestorLastDDLTimestampMap();
+ assertEquals(baseTableLastDDLTimestamp, map.get(baseTableKey));
+ assertEquals(1, basePTable.getIndexes().size());
+ map =
basePTable.getIndexes().get(0).getAncestorLastDDLTimestampMap();
+ assertEquals(baseTableLastDDLTimestamp, map.get(baseTableKey));
+
+ //tenant2 view
+ try (Connection tenant2Conn = getTenantConnection(tenant2)) {
+ map = PhoenixRuntime.getTable(tenant2Conn,
view3).getAncestorLastDDLTimestampMap();
+ assertEquals(1, map.size());
+ assertEquals(baseTableLastDDLTimestamp,
map.get(baseTableKey));
+ }
+ try (Connection tenant1Conn = getTenantConnection(tenant1)) {
+ //tenant1 view
+ PTable view1PTable = PhoenixRuntime.getTable(tenant1Conn,
view1);
+ map = view1PTable.getAncestorLastDDLTimestampMap();
+ assertEquals(1, map.size());
+ assertEquals(baseTableLastDDLTimestamp,
map.get(baseTableKey));
+ //tenant1 child view
+ PTableKey view1Key = new PTableKey(view1PTable.getTenantId(),
view1);
+ map = PhoenixRuntime.getTable(tenant1Conn,
view2).getAncestorLastDDLTimestampMap();
+ assertEquals(2, map.size());
+ assertEquals(baseTableLastDDLTimestamp,
map.get(baseTableKey));
+ assertEquals(view1PTable.getLastDDLTimestamp(),
map.get(view1Key));
+ //tenant1 child view index in cache and in child view PTable
+ PTable view2PTable = PhoenixRuntime.getTable(tenant1Conn,
view2);
+ PTableKey view2Key = new PTableKey(view2PTable.getTenantId(),
view2);
+ PTable index2PTable = PhoenixRuntime.getTable(tenant1Conn,
SchemaUtil.getTableName(SCHEMA3, index2));
+ map = index2PTable.getAncestorLastDDLTimestampMap();
+ assertEquals(baseTableLastDDLTimestamp,
map.get(baseTableKey));
+ assertEquals(view2PTable.getLastDDLTimestamp(),
map.get(view2Key));
+ assertEquals(2, view2PTable.getIndexes().size());
+ for (PTable index : view2PTable.getIndexes()) {
+ // inherited index
+ if (index.getTableName().getString().equals(index1)) {
+ map = index.getAncestorLastDDLTimestampMap();
+ assertEquals(baseTableLastDDLTimestamp,
map.get(baseTableKey));
+ } else {
+ // view index
+ map = index.getAncestorLastDDLTimestampMap();
+ assertEquals(baseTableLastDDLTimestamp,
map.get(baseTableKey));
+ assertEquals(view2PTable.getLastDDLTimestamp(),
map.get(view2Key));
+ }
+ }
+ }
+ }
+ }
+
+ private Connection getTenantConnection(String tenantId) throws
SQLException {
+ Properties tenantProps = new Properties();
+ tenantProps.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
+ return DriverManager.getConnection(getUrl(), tenantProps);
+ }
+
+ @Test
+ public void testAlterViewAndViewIndexMaxLookbackAgeFails() throws
Exception {
+ String schemaName = generateUniqueName();
+ String dataTableName = generateUniqueName();
+ String fullDataTableName = SchemaUtil.getTableName(schemaName,
dataTableName);
+ try(Connection conn = DriverManager.getConnection(getUrl());
+ Statement stmt = conn.createStatement()) {
+ String ddl = "CREATE TABLE " + fullDataTableName + " (ID VARCHAR
NOT NULL PRIMARY KEY, COL1 INTEGER)";
+ stmt.execute(ddl);
+ assertNull(queryTableLevelMaxLookbackAge(fullDataTableName));
+ String viewName = generateUniqueName();
+ String fullViewName = SchemaUtil.getTableName(schemaName,
viewName);
+ ddl = "CREATE VIEW " + fullViewName + " AS SELECT * FROM " +
fullDataTableName;
+ stmt.execute(ddl);
+ assertNull(queryTableLevelMaxLookbackAge(fullViewName));
+ String alterViewDdl = "ALTER VIEW " + fullViewName + " SET
MAX_LOOKBACK_AGE = 300";
+ SQLException err = assertThrows(SQLException.class, () ->
stmt.execute(alterViewDdl));
+ assertEquals(VIEW_WITH_PROPERTIES.getErrorCode(),
err.getErrorCode());
+ String viewIndexName = generateUniqueName();
+ String fullViewIndexName = SchemaUtil.getTableName(schemaName,
viewIndexName);
+ ddl = "CREATE INDEX " + viewIndexName + " ON " + fullViewName + "
(COL1)";
+ stmt.execute(ddl);
+ assertNull(queryTableLevelMaxLookbackAge(fullViewIndexName));
+ String alterViewIndexDdl = "ALTER INDEX " + viewIndexName + " ON
" + fullViewName +
+ " ACTIVE SET MAX_LOOKBACK_AGE = 300";
+ err = assertThrows(SQLException.class, () ->
stmt.execute(alterViewIndexDdl));
+
assertEquals(MAX_LOOKBACK_AGE_SUPPORTED_FOR_TABLES_ONLY.getErrorCode(),
err.getErrorCode());
+ }
+ }
+
private void assertPKs(ResultSet rs, String[] expectedPKs)
throws SQLException {
List<String> pkCols =
newArrayListWithExpectedSize(expectedPKs.length);
diff --cc phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
index 9018246cd5,76175e4dc9..18892db735
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
@@@ -142,10 -145,9 +147,11 @@@ import org.apache.phoenix.end2end.Paral
import org.apache.phoenix.end2end.ParallelStatsDisabledTest;
import org.apache.phoenix.end2end.ParallelStatsEnabledIT;
import org.apache.phoenix.end2end.ParallelStatsEnabledTest;
+import org.apache.phoenix.end2end.PhoenixRegionServerEndpointTestImpl;
+import org.apache.phoenix.end2end.ServerMetadataCacheTestImpl;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.exception.SQLExceptionInfo;
+ import org.apache.phoenix.hbase.index.IndexRegionObserver;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver;
