This is an automated email from the ASF dual-hosted git repository.
yanxinyi pushed a commit to branch 4.x
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x by this push:
new 09486b1 PHOENIX-6172 Updating VIEW_INDEX_ID column type and ts in
Syscat with a 4.16 upgrade script
09486b1 is described below
commit 09486b11327c4d399aa979cbb2acba6dbb763354
Author: Xinyi Yan <[email protected]>
AuthorDate: Fri Oct 16 20:17:01 2020 -0700
PHOENIX-6172 Updating VIEW_INDEX_ID column type and ts in Syscat with a
4.16 upgrade script
Signed-off-by: Xinyi Yan <[email protected]>
---
.../java/org/apache/phoenix/end2end/UpgradeIT.java | 75 ++++++++++++++++++++++
.../phoenix/query/ConnectionQueryServicesImpl.java | 25 ++++++++
.../java/org/apache/phoenix/util/UpgradeUtil.java | 39 +++++++++++
3 files changed, 139 insertions(+)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
index 6abf115..0f22da6 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
@@ -18,6 +18,10 @@
package org.apache.phoenix.end2end;
import static com.google.common.base.Preconditions.checkNotNull;
+import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID;
+import static org.apache.phoenix.query.QueryConstants.SYSTEM_SCHEMA_NAME;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
@@ -31,6 +35,7 @@ import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.sql.Types;
import java.util.List;
import java.util.Properties;
import java.util.Set;
@@ -42,7 +47,15 @@ import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.coprocessor.MetaDataProtocol;
@@ -64,6 +77,7 @@ import org.apache.phoenix.schema.PTable.LinkType;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.SequenceAllocation;
import org.apache.phoenix.schema.SequenceKey;
+import org.apache.phoenix.schema.types.PInteger;
import org.apache.phoenix.util.EnvironmentEdgeManager;
import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.PhoenixRuntime;
@@ -672,4 +686,65 @@ public class UpgradeIT extends ParallelStatsDisabledIT {
return key;
}
+ @Test
+ public void testUpgradeViewIndexIdDataType() throws Exception {
+ byte[] rowKey = SchemaUtil.getColumnKey(null,
+ SYSTEM_SCHEMA_NAME, SYSTEM_CATALOG_TABLE, VIEW_INDEX_ID,
+ PhoenixDatabaseMetaData.TABLE_FAMILY);
+ byte[] syscatBytes =
PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME.getBytes();
+ byte[] viewIndexIdTypeCellValueIn414 =
PInteger.INSTANCE.toBytes(Types.SMALLINT);
+ byte[] viewIndexIdTypeCellValueIn416 =
PInteger.INSTANCE.toBytes(Types.BIGINT);
+
+ try (PhoenixConnection conn = getConnection(false, null).
+ unwrap(PhoenixConnection.class)) {
+ // update the VIEW_INDEX_ID 0:DATAT_TYPE cell value to SMALLINT
+ // (4.14 and prior version is a SMALLINT column)
+ updateViewIndexIdColumnValue(rowKey, syscatBytes,
viewIndexIdTypeCellValueIn414);
+
assertTrue(UpgradeUtil.isUpdateViewIndexIdColumnDataTypeFromShortToLongNeeded(
+ conn, rowKey, syscatBytes));
+ verifyExpectedCellValue(rowKey, syscatBytes,
viewIndexIdTypeCellValueIn414);
+ // calling UpgradeUtil to mock the upgrade VIEW_INDEX_ID data type
to BIGINT
+ UpgradeUtil.updateViewIndexIdColumnDataTypeFromShortToLong(conn,
rowKey, syscatBytes);
+ verifyExpectedCellValue(rowKey, syscatBytes,
viewIndexIdTypeCellValueIn416);
+
assertFalse(UpgradeUtil.isUpdateViewIndexIdColumnDataTypeFromShortToLongNeeded(
+ conn, rowKey, syscatBytes));
+ } finally {
+ updateViewIndexIdColumnValue(rowKey, syscatBytes,
viewIndexIdTypeCellValueIn416);
+ }
+ }
+
+ private void updateViewIndexIdColumnValue(byte[] rowKey, byte[]
syscatBytes,
+ byte[] newColumnValue) throws
Exception {
+
+ try (PhoenixConnection conn =
+
DriverManager.getConnection(getUrl()).unwrap(PhoenixConnection.class);
+ Table sysTable = conn.getQueryServices().getTable(syscatBytes)) {
+ KeyValue viewIndexIdKV = new KeyValue(rowKey,
+ PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+ PhoenixDatabaseMetaData.DATA_TYPE_BYTES,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP,
+ newColumnValue);
+ Put viewIndexIdPut = new Put(rowKey);
+ viewIndexIdPut.add(viewIndexIdKV);
+ sysTable.put(viewIndexIdPut);
+ }
+ }
+
+ private void verifyExpectedCellValue(byte[] rowKey, byte[] syscatBytes,
+ byte[] expectedDateTypeBytes) throws
Exception {
+ try(PhoenixConnection conn = getConnection(false, null).
+ unwrap(PhoenixConnection.class);
+ Table sysTable = conn.getQueryServices().getTable(syscatBytes)) {
+ Scan s = new Scan();
+ s.setRowPrefixFilter(rowKey);
+ s.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+ PhoenixDatabaseMetaData.DATA_TYPE_BYTES);
+ ResultScanner scanner = sysTable.getScanner(s);
+ Result result= scanner.next();
+ Cell cell = result.getColumnLatestCell(
+ PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+ PhoenixDatabaseMetaData.DATA_TYPE_BYTES);
+ assertArrayEquals(expectedDateTypeBytes,
CellUtil.cloneValue(cell));
+ }
+ }
}
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index f9011bd..8fe3eec 100644
---
a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++
b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -60,6 +60,7 @@ import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTIONAL;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TTL_FOR_MUTEX;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID;
import static
org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HCONNECTIONS_COUNTER;
import static
org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_PHOENIX_CONNECTIONS_THROTTLED_COUNTER;
import static
org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_QUERY_SERVICES_COUNTER;
@@ -281,6 +282,7 @@ import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.ServerUtil;
+import org.apache.phoenix.util.StringUtil;
import org.apache.phoenix.util.TimeKeeper;
import org.apache.phoenix.util.UpgradeUtil;
import org.slf4j.Logger;
@@ -3725,6 +3727,29 @@ public class ConnectionQueryServicesImpl extends
DelegateQueryServices implement
MIN_SYSTEM_TABLE_TIMESTAMP_4_16_0,
PhoenixDatabaseMetaData.PHOENIX_TTL_HWM + " "
+ PInteger.INSTANCE.getSqlTypeName());
+
+ boolean isNamespaceMapping =
+ SchemaUtil.isNamespaceMappingEnabled(null,
getConfiguration());
+ String tableName = PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME;
+ if (isNamespaceMapping) {
+ tableName = tableName.replace(
+ QueryConstants.NAME_SEPARATOR,
+ QueryConstants.NAMESPACE_SEPARATOR);
+ }
+ byte[] tableBytes = StringUtil.toBytes(tableName);
+ byte[] rowKey = SchemaUtil.getColumnKey(null,
+ QueryConstants.SYSTEM_SCHEMA_NAME,
+ SYSTEM_CATALOG_TABLE, VIEW_INDEX_ID,
+ PhoenixDatabaseMetaData.TABLE_FAMILY);
+ if
(UpgradeUtil.isUpdateViewIndexIdColumnDataTypeFromShortToLongNeeded
+ (metaConnection, rowKey, tableBytes)) {
+ LOGGER.info("Updating VIEW_INDEX_ID data type to BIGINT.");
+ UpgradeUtil.updateViewIndexIdColumnDataTypeFromShortToLong(
+ metaConnection, rowKey, tableBytes
+ );
+ } else {
+ LOGGER.info("Updating VIEW_INDEX_ID data type is not needed.");
+ }
}
return metaConnection;
}
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
index 20a2a46..6a0e2ef 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
@@ -63,6 +63,7 @@ import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.sql.Types;
import java.text.Format;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
@@ -81,6 +82,7 @@ import javax.annotation.Nullable;
import com.google.common.base.Strings;
import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
@@ -2252,6 +2254,43 @@ public class UpgradeUtil {
}
}
+ public static boolean
isUpdateViewIndexIdColumnDataTypeFromShortToLongNeeded(
+ PhoenixConnection metaConnection, byte[] rowKey, byte[]
syscatBytes) {
+ try (Table sysTable =
metaConnection.getQueryServices().getTable(syscatBytes)) {
+ Scan s = new Scan();
+ s.setRowPrefixFilter(rowKey);
+ s.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+ PhoenixDatabaseMetaData.DATA_TYPE_BYTES);
+ ResultScanner scanner = sysTable.getScanner(s);
+ Result result = scanner.next();
+ Cell cell = result.getColumnLatestCell(
+ PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+ PhoenixDatabaseMetaData.DATA_TYPE_BYTES);
+ return Bytes.compareTo(CellUtil.cloneValue(cell),
+ PInteger.INSTANCE.toBytes(Types.SMALLINT)) == 0 ? true :
false;
+ } catch (Exception e) {
+ LOGGER.error(String.format("Checking VIEW_INDEX_ID data type for
upgrade failed: %s. ", e.getMessage()));
+ }
+ return false;
+ }
+
+ public static void updateViewIndexIdColumnDataTypeFromShortToLong(
+ PhoenixConnection metaConnection, byte[] rowKey, byte[]
syscatBytes) {
+ try(Table sysTable =
metaConnection.getQueryServices().getTable(syscatBytes)) {
+ KeyValue viewIndexIdKV = new KeyValue(rowKey,
+ PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+ PhoenixDatabaseMetaData.DATA_TYPE_BYTES,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP,
+ PInteger.INSTANCE.toBytes(Types.BIGINT));
+ Put viewIndexIdPut = new Put(rowKey);
+ viewIndexIdPut.add(viewIndexIdKV);
+ sysTable.put(viewIndexIdPut);
+ LOGGER.info("Updated VIEW_INDEX_ID data type from SMALLINT TO
BIGINT.");
+ } catch (Exception e) {
+ LOGGER.error(String.format("Upgrade/change VIEW_INDEX_ID data type
failed: %s. ",e.getMessage()));
+ }
+ }
+
private static void updateIndexesSequenceIfPresent(PhoenixConnection
connection, PTable dataTable)
throws SQLException {
PName tenantId = connection.getTenantId();
