This is an automated email from the ASF dual-hosted git repository. stoty pushed a commit to branch 5.1 in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/5.1 by this push: new 194db99 PHOENIX-6645 Remove unneccessary SCN related properties from SYSTEM tables on upgrade 194db99 is described below commit 194db993741798fd49d50145e0ba12fecf6b34af Author: Istvan Toth <st...@apache.org> AuthorDate: Thu Feb 17 18:18:01 2022 +0100 PHOENIX-6645 Remove unneccessary SCN related properties from SYSTEM tables on upgrade --- .../java/org/apache/phoenix/end2end/UpgradeIT.java | 59 ++++++++++++++++++++ .../phoenix/jdbc/PhoenixDatabaseMetaData.java | 2 + .../phoenix/query/ConnectionQueryServicesImpl.java | 62 +++++++++++++++++----- .../java/org/apache/phoenix/util/UpgradeUtil.java | 24 ++++++++- 4 files changed, 132 insertions(+), 15 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java index 274df5f..b4e62c9 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_FAMILY; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LAST_DDL_TIMESTAMP; @@ -43,7 +44,9 @@ import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.sql.Types; +import java.util.HashMap; import java.util.List; import java.util.Properties; import java.util.Set; @@ -57,12 +60,15 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.phoenix.schema.types.PInteger; import org.apache.phoenix.thirdparty.com.google.common.collect.Lists; import org.apache.phoenix.thirdparty.com.google.common.collect.Sets; @@ -77,6 +83,7 @@ import org.apache.phoenix.query.DelegateConnectionQueryServices; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PNameFactory; +import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTable.LinkType; import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.SequenceAllocation; @@ -358,6 +365,58 @@ public class UpgradeIT extends ParallelStatsDisabledIT { } } + @Test + public void testRemoveScnFromTaskAndLog() throws Exception { + PhoenixConnection conn = getConnection(false, null).unwrap(PhoenixConnection.class); + ConnectionQueryServicesImpl cqs = (ConnectionQueryServicesImpl)(conn.getQueryServices()); + //Set the SCN related properties on SYSTEM.STATS and SYSTEM.LOG + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("ALTER TABLE " + + PhoenixDatabaseMetaData.SYSTEM_LOG_NAME + " SET " + + KEEP_DELETED_CELLS + "='" + KeepDeletedCells.TRUE + "',\n" + + HConstants.VERSIONS + "='1000'\n"); + stmt.executeUpdate("ALTER TABLE " + + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + " SET " + + KEEP_DELETED_CELLS + "='" + KeepDeletedCells.TRUE + "',\n" + + HConstants.VERSIONS + "='1000'\n"); + } + //Check that the HBase tables reflect the change + PTable sysLogTable = PhoenixRuntime.getTable(conn, PhoenixDatabaseMetaData.SYSTEM_LOG_NAME); + TableDescriptor sysLogDesc = utility.getAdmin().getDescriptor(SchemaUtil.getPhysicalTableName( + PhoenixDatabaseMetaData.SYSTEM_LOG_NAME, cqs.getProps())); + assertEquals(KeepDeletedCells.TRUE, sysLogDesc.getColumnFamily( + SchemaUtil.getEmptyColumnFamily(sysLogTable)).getKeepDeletedCells()); + assertEquals(1000, sysLogDesc.getColumnFamily( + SchemaUtil.getEmptyColumnFamily(sysLogTable)).getMaxVersions()); + + PTable sysStatsTable = PhoenixRuntime.getTable(conn, PhoenixDatabaseMetaData.SYSTEM_STATS_NAME); + TableDescriptor sysStatsDesc = utility.getAdmin().getDescriptor(SchemaUtil.getPhysicalTableName( + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME, cqs.getProps())); + assertEquals(KeepDeletedCells.TRUE, sysStatsDesc.getColumnFamily( + SchemaUtil.getEmptyColumnFamily(sysStatsTable)).getKeepDeletedCells()); + assertEquals(1000, sysStatsDesc.getColumnFamily( + SchemaUtil.getEmptyColumnFamily(sysStatsTable)).getMaxVersions()); + + //now call the upgrade code + cqs.upgradeSystemLog(conn, new HashMap<String, String>()); + cqs.upgradeSystemStats(conn, new HashMap<String, String>()); + + //Check that HBase tables are fixed + sysLogDesc = utility.getAdmin().getDescriptor(SchemaUtil.getPhysicalTableName( + PhoenixDatabaseMetaData.SYSTEM_LOG_NAME, cqs.getProps())); + assertEquals(KeepDeletedCells.FALSE, sysLogDesc.getColumnFamily( + SchemaUtil.getEmptyColumnFamily(sysLogTable)).getKeepDeletedCells()); + assertEquals(1, sysLogDesc.getColumnFamily( + SchemaUtil.getEmptyColumnFamily(sysLogTable)).getMaxVersions()); + + sysStatsDesc = utility.getAdmin().getDescriptor(SchemaUtil.getPhysicalTableName( + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME, cqs.getProps())); + assertEquals(KeepDeletedCells.FALSE, sysStatsDesc.getColumnFamily( + SchemaUtil.getEmptyColumnFamily(sysStatsTable)).getKeepDeletedCells()); + assertEquals(1, sysStatsDesc.getColumnFamily( + SchemaUtil.getEmptyColumnFamily(sysStatsTable)).getMaxVersions()); + } + private Set<String> getChildLinks(Connection conn) throws SQLException { ResultSet rs = conn.createStatement().executeQuery( diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java index de5ba5d..4866d40 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java @@ -401,6 +401,8 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData { public static final TableName SYSTEM_TASK_HBASE_TABLE_NAME = TableName.valueOf(SYSTEM_TASK_NAME); //SYSTEM:LOG public static final String SYSTEM_LOG_TABLE = "LOG"; + public static final String SYSTEM_LOG_NAME = + SchemaUtil.getTableName(SYSTEM_CATALOG_SCHEMA, SYSTEM_LOG_TABLE); public static final String QUERY_ID = "QUERY_ID"; public static final String USER = "USER"; public static final String CLIENT_IP = "CLIENT_IP"; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index 220ec14..3487093 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -4079,7 +4079,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement metaConnection = upgradeSystemTask(metaConnection, systemTableToSnapshotMap); metaConnection = upgradeSystemFunction(metaConnection); - metaConnection = upgradeSystemLog(metaConnection); + metaConnection = upgradeSystemLog(metaConnection, systemTableToSnapshotMap); return upgradeSystemMutex(metaConnection); } @@ -4179,9 +4179,11 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement tableName); } - private PhoenixConnection upgradeSystemStats( + @VisibleForTesting + public PhoenixConnection upgradeSystemStats( PhoenixConnection metaConnection, - Map<String, String> systemTableToSnapshotMap) throws SQLException { + Map<String, String> systemTableToSnapshotMap) throws + SQLException, org.apache.hadoop.hbase.TableNotFoundException, IOException { try (Statement statement = metaConnection.createStatement()) { statement.executeUpdate(QueryConstants.CREATE_STATS_TABLE_METADATA); } catch (NewerTableAlreadyExistsException ignored) { @@ -4213,6 +4215,22 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0); clearCache(); } + if (UpgradeUtil.tableHasKeepDeleted( + metaConnection, PhoenixDatabaseMetaData.SYSTEM_STATS_NAME)) { + try (Statement stmt = metaConnection.createStatement()){ + stmt.executeUpdate("ALTER TABLE " + + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + " SET " + + KEEP_DELETED_CELLS + "='" + KeepDeletedCells.FALSE + "'"); + } + } + if (UpgradeUtil.tableHasMaxVersions( + metaConnection, PhoenixDatabaseMetaData.SYSTEM_STATS_NAME)) { + try (Statement stmt = metaConnection.createStatement()){ + stmt.executeUpdate("ALTER TABLE " + + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + " SET " + + HConstants.VERSIONS + "='1'"); + } + } } return metaConnection; } @@ -4267,7 +4285,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement isTableDescUpdated = true; } if (!tableDescriptorBuilder.build().hasCoprocessor( - TaskMetaDataEndpoint.class.getName())) { + TaskMetaDataEndpoint.class.getName())) { int priority = props.getInt( QueryServices.COPROCESSOR_PRIORITY_ATTRIB, QueryServicesOptions.DEFAULT_COPROCESSOR_PRIORITY); @@ -4303,16 +4321,32 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement return metaConnection; } - private PhoenixConnection upgradeSystemLog(PhoenixConnection metaConnection) - throws SQLException { - try { - metaConnection.createStatement().executeUpdate(getLogTableDDL()); - } catch (TableAlreadyExistsException ignored) { - // Since we are not performing any action as part of upgrading - // SYSTEM.LOG, we don't need to take snapshot as of this - // writing. However, if need arises to perform significant - // update, we should take snapshot just like other system tables. - // e.g usages of takeSnapshotOfSysTable() + @VisibleForTesting + public PhoenixConnection upgradeSystemLog(PhoenixConnection metaConnection, + Map<String, String> systemTableToSnapshotMap) + throws SQLException, org.apache.hadoop.hbase.TableNotFoundException, IOException { + try (Statement statement = metaConnection.createStatement()) { + statement.executeUpdate(getLogTableDDL()); + } catch (NewerTableAlreadyExistsException ignored) { + } catch (TableAlreadyExistsException e) { + // take snapshot first + takeSnapshotOfSysTable(systemTableToSnapshotMap, e); + if(UpgradeUtil.tableHasKeepDeleted( + metaConnection, PhoenixDatabaseMetaData.SYSTEM_LOG_NAME) ) { + try (Statement stmt = metaConnection.createStatement()) { + stmt.executeUpdate("ALTER TABLE " + + PhoenixDatabaseMetaData.SYSTEM_LOG_NAME + " SET " + + KEEP_DELETED_CELLS + "='" + KeepDeletedCells.FALSE + "'"); + } + } + if (UpgradeUtil.tableHasMaxVersions( + metaConnection, PhoenixDatabaseMetaData.SYSTEM_LOG_NAME)) { + try (Statement stmt = metaConnection.createStatement()) { + stmt.executeUpdate("ALTER TABLE " + + PhoenixDatabaseMetaData.SYSTEM_LOG_NAME + " SET " + + HConstants.VERSIONS + "='1'"); + } + } } return metaConnection; } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java index 6495db5..0fe78e3 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java @@ -46,7 +46,6 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCH import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME_BYTES; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_CAT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM; @@ -91,6 +90,7 @@ import org.apache.phoenix.thirdparty.com.google.common.base.Strings; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; @@ -2629,4 +2629,26 @@ public class UpgradeUtil { } LOGGER.info("Setting DDL timestamps for tables and views is complete"); } + + public static boolean tableHasKeepDeleted(PhoenixConnection conn, String pTableName) + throws SQLException, org.apache.hadoop.hbase.TableNotFoundException, IOException { + ConnectionQueryServices cqs = conn.getQueryServices(); + Admin admin = cqs.getAdmin(); + PTable table = PhoenixRuntime.getTable(conn, pTableName); + TableDescriptor tableDesc = admin.getDescriptor(SchemaUtil.getPhysicalTableName( + pTableName, cqs.getProps())); + return KeepDeletedCells.TRUE.equals(tableDesc.getColumnFamily( + SchemaUtil.getEmptyColumnFamily(table)).getKeepDeletedCells()); + } + + public static boolean tableHasMaxVersions(PhoenixConnection conn, String pTableName) + throws SQLException, org.apache.hadoop.hbase.TableNotFoundException, IOException { + ConnectionQueryServices cqs = conn.getQueryServices(); + Admin admin = cqs.getAdmin(); + PTable table = PhoenixRuntime.getTable(conn, pTableName); + TableDescriptor tableDesc = admin.getDescriptor(SchemaUtil.getPhysicalTableName( + pTableName, cqs.getProps())); + return tableDesc.getColumnFamily( + SchemaUtil.getEmptyColumnFamily(table)).getMaxVersions() > 1; + } }