This is an automated email from the ASF dual-hosted git repository. stoty pushed a commit to branch 4.x in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x by this push: new 42baa30 PHOENIX-6645 Remove unneccessary SCN related properties from SYSTEM tables on upgrade 42baa30 is described below commit 42baa30516a9979553263aca8f6468314e27cadc Author: Istvan Toth <st...@apache.org> AuthorDate: Thu Feb 17 18:18:01 2022 +0100 PHOENIX-6645 Remove unneccessary SCN related properties from SYSTEM tables on upgrade --- .../java/org/apache/phoenix/end2end/UpgradeIT.java | 59 +++++++++++++++++++++ .../phoenix/jdbc/PhoenixDatabaseMetaData.java | 2 + .../phoenix/query/ConnectionQueryServicesImpl.java | 60 +++++++++++++++++----- .../java/org/apache/phoenix/util/UpgradeUtil.java | 25 ++++++++- 4 files changed, 131 insertions(+), 15 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java index 29c72c2..3a1b8f1 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java @@ -18,6 +18,7 @@ package org.apache.phoenix.end2end; import static org.apache.phoenix.thirdparty.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.hadoop.hbase.HColumnDescriptor.KEEP_DELETED_CELLS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_FAMILY; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LAST_DDL_TIMESTAMP; @@ -43,7 +44,9 @@ import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.sql.Types; +import java.util.HashMap; import java.util.List; import java.util.Properties; import java.util.Set; @@ -60,6 +63,9 @@ import org.apache.phoenix.thirdparty.com.google.common.collect.Sets; import org.apache.phoenix.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; @@ -78,6 +84,7 @@ import org.apache.phoenix.query.DelegateConnectionQueryServices; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PNameFactory; +import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTable.LinkType; import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.SequenceAllocation; @@ -358,6 +365,58 @@ public class UpgradeIT extends ParallelStatsDisabledIT { } } + @Test + public void testRemoveScnFromTaskAndLog() throws Exception { + PhoenixConnection conn = getConnection(false, null).unwrap(PhoenixConnection.class); + ConnectionQueryServicesImpl cqs = (ConnectionQueryServicesImpl)(conn.getQueryServices()); + //Set the SCN related properties on SYSTEM.STATS and SYSTEM.LOG + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("ALTER TABLE " + + PhoenixDatabaseMetaData.SYSTEM_LOG_NAME + " SET " + + KEEP_DELETED_CELLS + "='" + KeepDeletedCells.TRUE + "',\n" + + HConstants.VERSIONS + "='1000'\n"); + stmt.executeUpdate("ALTER TABLE " + + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + " SET " + + KEEP_DELETED_CELLS + "='" + KeepDeletedCells.TRUE + "',\n" + + HConstants.VERSIONS + "='1000'\n"); + } + //Check that the HBase tables reflect the change + PTable sysLogTable = PhoenixRuntime.getTable(conn, PhoenixDatabaseMetaData.SYSTEM_LOG_NAME); + HTableDescriptor sysLogDesc = utility.getHBaseAdmin().getTableDescriptor(SchemaUtil.getPhysicalTableName( + PhoenixDatabaseMetaData.SYSTEM_LOG_NAME, cqs.getProps())); + assertEquals(KeepDeletedCells.TRUE, sysLogDesc.getFamily( + SchemaUtil.getEmptyColumnFamily(sysLogTable)).getKeepDeletedCells()); + assertEquals(1000, sysLogDesc.getFamily( + SchemaUtil.getEmptyColumnFamily(sysLogTable)).getMaxVersions()); + + PTable sysStatsTable = PhoenixRuntime.getTable(conn, PhoenixDatabaseMetaData.SYSTEM_STATS_NAME); + HTableDescriptor sysStatsDesc = utility.getHBaseAdmin().getTableDescriptor(SchemaUtil.getPhysicalTableName( + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME, cqs.getProps())); + assertEquals(KeepDeletedCells.TRUE, sysStatsDesc.getFamily( + SchemaUtil.getEmptyColumnFamily(sysStatsTable)).getKeepDeletedCells()); + assertEquals(1000, sysStatsDesc.getFamily( + SchemaUtil.getEmptyColumnFamily(sysStatsTable)).getMaxVersions()); + + //now call the upgrade code + cqs.upgradeSystemLog(conn, new HashMap<String, String>()); + cqs.upgradeSystemStats(conn, new HashMap<String, String>()); + + //Check that HBase tables are fixed + sysLogDesc = utility.getHBaseAdmin().getTableDescriptor(SchemaUtil.getPhysicalTableName( + PhoenixDatabaseMetaData.SYSTEM_LOG_NAME, cqs.getProps())); + assertEquals(KeepDeletedCells.FALSE, sysLogDesc.getFamily( + SchemaUtil.getEmptyColumnFamily(sysLogTable)).getKeepDeletedCells()); + assertEquals(1, sysLogDesc.getFamily( + SchemaUtil.getEmptyColumnFamily(sysLogTable)).getMaxVersions()); + + sysStatsDesc = utility.getHBaseAdmin().getTableDescriptor(SchemaUtil.getPhysicalTableName( + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME, cqs.getProps())); + assertEquals(KeepDeletedCells.FALSE, sysStatsDesc.getFamily( + SchemaUtil.getEmptyColumnFamily(sysStatsTable)).getKeepDeletedCells()); + assertEquals(1, sysStatsDesc.getFamily( + SchemaUtil.getEmptyColumnFamily(sysStatsTable)).getMaxVersions()); + } + private Set<String> getChildLinks(Connection conn) throws SQLException { ResultSet rs = conn.createStatement().executeQuery( diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java index 6861ecc..992ac0f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java @@ -411,6 +411,8 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData { //SYSTEM:LOG public static final String SYSTEM_LOG_TABLE = "LOG"; + public static final String SYSTEM_LOG_NAME = + SchemaUtil.getTableName(SYSTEM_CATALOG_SCHEMA, SYSTEM_LOG_TABLE); public static final String QUERY_ID = "QUERY_ID"; public static final String USER = "USER"; public static final String CLIENT_IP = "CLIENT_IP"; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index d8da304..9b3497d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -4120,7 +4120,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement systemTableToSnapshotMap); metaConnection = upgradeSystemFunction(metaConnection); metaConnection = upgradeSystemTransform(metaConnection, systemTableToSnapshotMap); - metaConnection = upgradeSystemLog(metaConnection); + metaConnection = upgradeSystemLog(metaConnection, systemTableToSnapshotMap); return upgradeSystemMutex(metaConnection); } @@ -4219,9 +4219,11 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement tableName); } - private PhoenixConnection upgradeSystemStats( + @VisibleForTesting + public PhoenixConnection upgradeSystemStats( PhoenixConnection metaConnection, - Map<String, String> systemTableToSnapshotMap) throws SQLException { + Map<String, String> systemTableToSnapshotMap) throws + SQLException, org.apache.hadoop.hbase.TableNotFoundException, IOException { try (Statement statement = metaConnection.createStatement()) { statement.executeUpdate(QueryConstants.CREATE_STATS_TABLE_METADATA); } catch (NewerTableAlreadyExistsException ignored) { @@ -4253,6 +4255,22 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0); clearCache(); } + if (UpgradeUtil.tableHasKeepDeleted( + metaConnection, PhoenixDatabaseMetaData.SYSTEM_STATS_NAME)) { + try (Statement stmt = metaConnection.createStatement()){ + stmt.executeUpdate("ALTER TABLE " + + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + " SET " + + KEEP_DELETED_CELLS + "='" + KeepDeletedCells.FALSE + "'"); + } + } + if (UpgradeUtil.tableHasMaxVersions( + metaConnection, PhoenixDatabaseMetaData.SYSTEM_STATS_NAME)) { + try (Statement stmt = metaConnection.createStatement()){ + stmt.executeUpdate("ALTER TABLE " + + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + " SET " + + HConstants.VERSIONS + "='1'"); + } + } } return metaConnection; } @@ -4351,16 +4369,32 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement return metaConnection; } - private PhoenixConnection upgradeSystemLog(PhoenixConnection metaConnection) - throws SQLException { - try { - metaConnection.createStatement().executeUpdate(getLogTableDDL()); - } catch (TableAlreadyExistsException ignored) { - // Since we are not performing any action as part of upgrading - // SYSTEM.LOG, we don't need to take snapshot as of this - // writing. However, if need arises to perform significant - // update, we should take snapshot just like other system tables. - // e.g usages of takeSnapshotOfSysTable() + @VisibleForTesting + public PhoenixConnection upgradeSystemLog(PhoenixConnection metaConnection, + Map<String, String> systemTableToSnapshotMap) + throws SQLException, org.apache.hadoop.hbase.TableNotFoundException, IOException { + try (Statement statement = metaConnection.createStatement()) { + statement.executeUpdate(getLogTableDDL()); + } catch (NewerTableAlreadyExistsException ignored) { + } catch (TableAlreadyExistsException e) { + // take snapshot first + takeSnapshotOfSysTable(systemTableToSnapshotMap, e); + if(UpgradeUtil.tableHasKeepDeleted( + metaConnection, PhoenixDatabaseMetaData.SYSTEM_LOG_NAME) ) { + try (Statement stmt = metaConnection.createStatement()) { + stmt.executeUpdate("ALTER TABLE " + + PhoenixDatabaseMetaData.SYSTEM_LOG_NAME + " SET " + + KEEP_DELETED_CELLS + "='" + KeepDeletedCells.FALSE + "'"); + } + } + if (UpgradeUtil.tableHasMaxVersions( + metaConnection, PhoenixDatabaseMetaData.SYSTEM_LOG_NAME)) { + try (Statement stmt = metaConnection.createStatement()) { + stmt.executeUpdate("ALTER TABLE " + + PhoenixDatabaseMetaData.SYSTEM_LOG_NAME + " SET " + + HConstants.VERSIONS + "='1'"); + } + } } return metaConnection; } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java index 435db26..f054cda 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java @@ -46,7 +46,6 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCH import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME_BYTES; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_CAT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM; @@ -67,7 +66,6 @@ import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Timestamp; import java.sql.Types; import java.text.Format; import java.text.SimpleDateFormat; @@ -92,6 +90,7 @@ import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; @@ -2641,4 +2640,26 @@ public class UpgradeUtil { } LOGGER.info("Setting DDL timestamps for tables and views is complete"); } + + public static boolean tableHasKeepDeleted(PhoenixConnection conn, String pTableName) + throws SQLException, org.apache.hadoop.hbase.TableNotFoundException, IOException { + ConnectionQueryServices cqs = conn.getQueryServices(); + Admin admin = cqs.getAdmin(); + PTable table = PhoenixRuntime.getTable(conn, pTableName); + HTableDescriptor tableDesc = admin.getTableDescriptor(SchemaUtil.getPhysicalTableName( + pTableName, cqs.getProps())); + return KeepDeletedCells.TRUE.equals(tableDesc.getFamily( + SchemaUtil.getEmptyColumnFamily(table)).getKeepDeletedCells()); + } + + public static boolean tableHasMaxVersions(PhoenixConnection conn, String pTableName) + throws SQLException, org.apache.hadoop.hbase.TableNotFoundException, IOException { + ConnectionQueryServices cqs = conn.getQueryServices(); + Admin admin = cqs.getAdmin(); + PTable table = PhoenixRuntime.getTable(conn, pTableName); + HTableDescriptor tableDesc = admin.getTableDescriptor(SchemaUtil.getPhysicalTableName( + pTableName, cqs.getProps())); + return tableDesc.getFamily( + SchemaUtil.getEmptyColumnFamily(table)).getMaxVersions() > 1; + } }