This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push:
new 47887b3 PHOENIX-6645 Remove unneccessary SCN related properties from
SYSTEM tables on upgrade
47887b3 is described below
commit 47887b3cefeb22ef70d8aba4f181f8704e553a79
Author: Istvan Toth <st...@apache.org>
AuthorDate: Thu Feb 17 18:18:01 2022 +0100
PHOENIX-6645 Remove unneccessary SCN related properties from SYSTEM tables
on upgrade
---
.../java/org/apache/phoenix/end2end/UpgradeIT.java | 59 ++++++++++++++++++++
.../phoenix/jdbc/PhoenixDatabaseMetaData.java | 2 +
.../phoenix/query/ConnectionQueryServicesImpl.java | 62 +++++++++++++++++-----
.../java/org/apache/phoenix/util/UpgradeUtil.java | 25 ++++++++-
4 files changed, 132 insertions(+), 16 deletions(-)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
index 274df5f..b4e62c9 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
@@ -17,6 +17,7 @@
*/
package org.apache.phoenix.end2end;
+import static
org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_FAMILY;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME;
import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LAST_DDL_TIMESTAMP;
@@ -43,7 +44,9 @@ import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.sql.Statement;
import java.sql.Types;
+import java.util.HashMap;
import java.util.List;
import java.util.Properties;
import java.util.Set;
@@ -57,12 +60,15 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.phoenix.schema.types.PInteger;
import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
import org.apache.phoenix.thirdparty.com.google.common.collect.Sets;
@@ -77,6 +83,7 @@ import
org.apache.phoenix.query.DelegateConnectionQueryServices;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PNameFactory;
+import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTable.LinkType;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.SequenceAllocation;
@@ -358,6 +365,58 @@ public class UpgradeIT extends ParallelStatsDisabledIT {
}
}
+ @Test
+ public void testRemoveScnFromTaskAndLog() throws Exception {
+ PhoenixConnection conn = getConnection(false,
null).unwrap(PhoenixConnection.class);
+ ConnectionQueryServicesImpl cqs =
(ConnectionQueryServicesImpl)(conn.getQueryServices());
+ //Set the SCN related properties on SYSTEM.STATS and SYSTEM.LOG
+ try (Statement stmt = conn.createStatement()) {
+ stmt.executeUpdate("ALTER TABLE " +
+ PhoenixDatabaseMetaData.SYSTEM_LOG_NAME + " SET " +
+ KEEP_DELETED_CELLS + "='" + KeepDeletedCells.TRUE + "',\n"
+
+ HConstants.VERSIONS + "='1000'\n");
+ stmt.executeUpdate("ALTER TABLE " +
+ PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + " SET " +
+ KEEP_DELETED_CELLS + "='" + KeepDeletedCells.TRUE + "',\n"
+
+ HConstants.VERSIONS + "='1000'\n");
+ }
+ //Check that the HBase tables reflect the change
+ PTable sysLogTable = PhoenixRuntime.getTable(conn,
PhoenixDatabaseMetaData.SYSTEM_LOG_NAME);
+ TableDescriptor sysLogDesc =
utility.getAdmin().getDescriptor(SchemaUtil.getPhysicalTableName(
+ PhoenixDatabaseMetaData.SYSTEM_LOG_NAME, cqs.getProps()));
+ assertEquals(KeepDeletedCells.TRUE, sysLogDesc.getColumnFamily(
+
SchemaUtil.getEmptyColumnFamily(sysLogTable)).getKeepDeletedCells());
+ assertEquals(1000, sysLogDesc.getColumnFamily(
+
SchemaUtil.getEmptyColumnFamily(sysLogTable)).getMaxVersions());
+
+ PTable sysStatsTable = PhoenixRuntime.getTable(conn,
PhoenixDatabaseMetaData.SYSTEM_STATS_NAME);
+ TableDescriptor sysStatsDesc =
utility.getAdmin().getDescriptor(SchemaUtil.getPhysicalTableName(
+ PhoenixDatabaseMetaData.SYSTEM_STATS_NAME, cqs.getProps()));
+ assertEquals(KeepDeletedCells.TRUE, sysStatsDesc.getColumnFamily(
+
SchemaUtil.getEmptyColumnFamily(sysStatsTable)).getKeepDeletedCells());
+ assertEquals(1000, sysStatsDesc.getColumnFamily(
+ SchemaUtil.getEmptyColumnFamily(sysStatsTable)).getMaxVersions());
+
+ //now call the upgrade code
+ cqs.upgradeSystemLog(conn, new HashMap<String, String>());
+ cqs.upgradeSystemStats(conn, new HashMap<String, String>());
+
+ //Check that HBase tables are fixed
+ sysLogDesc =
utility.getAdmin().getDescriptor(SchemaUtil.getPhysicalTableName(
+ PhoenixDatabaseMetaData.SYSTEM_LOG_NAME, cqs.getProps()));
+ assertEquals(KeepDeletedCells.FALSE, sysLogDesc.getColumnFamily(
+
SchemaUtil.getEmptyColumnFamily(sysLogTable)).getKeepDeletedCells());
+ assertEquals(1, sysLogDesc.getColumnFamily(
+
SchemaUtil.getEmptyColumnFamily(sysLogTable)).getMaxVersions());
+
+ sysStatsDesc =
utility.getAdmin().getDescriptor(SchemaUtil.getPhysicalTableName(
+ PhoenixDatabaseMetaData.SYSTEM_STATS_NAME, cqs.getProps()));
+ assertEquals(KeepDeletedCells.FALSE, sysStatsDesc.getColumnFamily(
+
SchemaUtil.getEmptyColumnFamily(sysStatsTable)).getKeepDeletedCells());
+ assertEquals(1, sysStatsDesc.getColumnFamily(
+ SchemaUtil.getEmptyColumnFamily(sysStatsTable)).getMaxVersions());
+ }
+
private Set<String> getChildLinks(Connection conn) throws SQLException {
ResultSet rs =
conn.createStatement().executeQuery(
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index eba2f1a..8c56948 100644
---
a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++
b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -428,6 +428,8 @@ public class PhoenixDatabaseMetaData implements
DatabaseMetaData {
//SYSTEM:LOG
public static final String SYSTEM_LOG_TABLE = "LOG";
+ public static final String SYSTEM_LOG_NAME =
+ SchemaUtil.getTableName(SYSTEM_CATALOG_SCHEMA, SYSTEM_LOG_TABLE);
public static final String QUERY_ID = "QUERY_ID";
public static final String USER = "USER";
public static final String CLIENT_IP = "CLIENT_IP";
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index ea25cff..966c973 100644
---
a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++
b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -4152,7 +4152,7 @@ public class ConnectionQueryServicesImpl extends
DelegateQueryServices implement
systemTableToSnapshotMap);
metaConnection = upgradeSystemFunction(metaConnection);
metaConnection = upgradeSystemTransform(metaConnection,
systemTableToSnapshotMap);
- metaConnection = upgradeSystemLog(metaConnection);
+ metaConnection = upgradeSystemLog(metaConnection,
systemTableToSnapshotMap);
return upgradeSystemMutex(metaConnection);
}
@@ -4252,9 +4252,11 @@ public class ConnectionQueryServicesImpl extends
DelegateQueryServices implement
tableName);
}
- private PhoenixConnection upgradeSystemStats(
+ @VisibleForTesting
+ public PhoenixConnection upgradeSystemStats(
PhoenixConnection metaConnection,
- Map<String, String> systemTableToSnapshotMap) throws SQLException {
+ Map<String, String> systemTableToSnapshotMap) throws
+ SQLException, org.apache.hadoop.hbase.TableNotFoundException,
IOException {
try (Statement statement = metaConnection.createStatement()) {
statement.executeUpdate(QueryConstants.CREATE_STATS_TABLE_METADATA);
} catch (NewerTableAlreadyExistsException ignored) {
@@ -4286,6 +4288,22 @@ public class ConnectionQueryServicesImpl extends
DelegateQueryServices implement
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0);
clearCache();
}
+ if (UpgradeUtil.tableHasKeepDeleted(
+ metaConnection, PhoenixDatabaseMetaData.SYSTEM_STATS_NAME)) {
+ try (Statement stmt = metaConnection.createStatement()){
+ stmt.executeUpdate("ALTER TABLE "
+ + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + "
SET "
+ + KEEP_DELETED_CELLS + "='" +
KeepDeletedCells.FALSE + "'");
+ }
+ }
+ if (UpgradeUtil.tableHasMaxVersions(
+ metaConnection, PhoenixDatabaseMetaData.SYSTEM_STATS_NAME)) {
+ try (Statement stmt = metaConnection.createStatement()){
+ stmt.executeUpdate("ALTER TABLE "
+ + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + "
SET "
+ + HConstants.VERSIONS + "='1'");
+ }
+ }
}
return metaConnection;
}
@@ -4340,7 +4358,7 @@ public class ConnectionQueryServicesImpl extends
DelegateQueryServices implement
isTableDescUpdated = true;
}
if (!tableDescriptorBuilder.build().hasCoprocessor(
- TaskMetaDataEndpoint.class.getName())) {
+ TaskMetaDataEndpoint.class.getName())) {
int priority = props.getInt(
QueryServices.COPROCESSOR_PRIORITY_ATTRIB,
QueryServicesOptions.DEFAULT_COPROCESSOR_PRIORITY);
@@ -4388,16 +4406,32 @@ public class ConnectionQueryServicesImpl extends
DelegateQueryServices implement
return metaConnection;
}
- private PhoenixConnection upgradeSystemLog(PhoenixConnection
metaConnection)
- throws SQLException {
- try {
- metaConnection.createStatement().executeUpdate(getLogTableDDL());
- } catch (TableAlreadyExistsException ignored) {
- // Since we are not performing any action as part of upgrading
- // SYSTEM.LOG, we don't need to take snapshot as of this
- // writing. However, if need arises to perform significant
- // update, we should take snapshot just like other system tables.
- // e.g usages of takeSnapshotOfSysTable()
+ @VisibleForTesting
+ public PhoenixConnection upgradeSystemLog(PhoenixConnection metaConnection,
+ Map<String, String> systemTableToSnapshotMap)
+ throws SQLException, org.apache.hadoop.hbase.TableNotFoundException,
IOException {
+ try (Statement statement = metaConnection.createStatement()) {
+ statement.executeUpdate(getLogTableDDL());
+ } catch (NewerTableAlreadyExistsException ignored) {
+ } catch (TableAlreadyExistsException e) {
+ // take snapshot first
+ takeSnapshotOfSysTable(systemTableToSnapshotMap, e);
+ if(UpgradeUtil.tableHasKeepDeleted(
+ metaConnection, PhoenixDatabaseMetaData.SYSTEM_LOG_NAME) ) {
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.executeUpdate("ALTER TABLE " +
+ PhoenixDatabaseMetaData.SYSTEM_LOG_NAME + " SET " +
+ KEEP_DELETED_CELLS + "='" + KeepDeletedCells.FALSE
+ "'");
+ }
+ }
+ if (UpgradeUtil.tableHasMaxVersions(
+ metaConnection, PhoenixDatabaseMetaData.SYSTEM_LOG_NAME)) {
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.executeUpdate("ALTER TABLE " +
+ PhoenixDatabaseMetaData.SYSTEM_LOG_NAME + " SET " +
+ HConstants.VERSIONS + "='1'");
+ }
+ }
}
return metaConnection;
}
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
index 04156f3..4285581 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
@@ -46,7 +46,6 @@ import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCH
import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE;
import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE_BYTES;
import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME_BYTES;
-import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_CAT;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
@@ -67,7 +66,6 @@ import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
-import java.sql.Timestamp;
import java.sql.Types;
import java.text.Format;
import java.text.SimpleDateFormat;
@@ -92,6 +90,7 @@ import
org.apache.phoenix.thirdparty.com.google.common.base.Strings;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
@@ -2652,4 +2651,26 @@ public class UpgradeUtil {
}
LOGGER.info("Setting DDL timestamps for tables and views is complete");
}
+
+ public static boolean tableHasKeepDeleted(PhoenixConnection conn, String
pTableName)
+ throws SQLException,
org.apache.hadoop.hbase.TableNotFoundException, IOException {
+ ConnectionQueryServices cqs = conn.getQueryServices();
+ Admin admin = cqs.getAdmin();
+ PTable table = PhoenixRuntime.getTable(conn, pTableName);
+ TableDescriptor tableDesc =
admin.getDescriptor(SchemaUtil.getPhysicalTableName(
+ pTableName, cqs.getProps()));
+ return KeepDeletedCells.TRUE.equals(tableDesc.getColumnFamily(
+ SchemaUtil.getEmptyColumnFamily(table)).getKeepDeletedCells());
+ }
+
+ public static boolean tableHasMaxVersions(PhoenixConnection conn, String
pTableName)
+ throws SQLException,
org.apache.hadoop.hbase.TableNotFoundException, IOException {
+ ConnectionQueryServices cqs = conn.getQueryServices();
+ Admin admin = cqs.getAdmin();
+ PTable table = PhoenixRuntime.getTable(conn, pTableName);
+ TableDescriptor tableDesc =
admin.getDescriptor(SchemaUtil.getPhysicalTableName(
+ pTableName, cqs.getProps()));
+ return tableDesc.getColumnFamily(
+ SchemaUtil.getEmptyColumnFamily(table)).getMaxVersions() > 1;
+ }
}
