PHOENIX-1641 Addendum to the previous commit
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/78d91d11 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/78d91d11 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/78d91d11 Branch: refs/heads/calcite Commit: 78d91d11e52acf54486c94ad8a0ec7dda2c9eb21 Parents: 2b8e663 Author: Samarth <samarth.j...@salesforce.com> Authored: Sat Feb 7 10:42:44 2015 -0800 Committer: Samarth <samarth.j...@salesforce.com> Committed: Sat Feb 7 10:42:44 2015 -0800 ---------------------------------------------------------------------- .../phoenix/coprocessor/MetaDataProtocol.java | 10 ++-- .../query/ConnectionQueryServicesImpl.java | 49 +++++++++++--------- .../org/apache/phoenix/util/UpgradeUtil.java | 12 +++-- 3 files changed, 42 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/78d91d11/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java index 357ebdd..be5fb4d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java @@ -60,13 +60,17 @@ public abstract class MetaDataProtocol extends MetaDataService { public static final long MIN_TABLE_TIMESTAMP = 0; - // Incremented from 3 to 4 to salt the sequence table in 3.2/4.2 - // Incremented from 5 to 6 with the addition of the STORE_NULLS table option + // Incremented from 5 to 6 with the addition of the STORE_NULLS table option in 4.3 public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_TABLE_TIMESTAMP + 6; public static final int DEFAULT_MAX_META_DATA_VERSIONS = 1000; public static final int DEFAULT_MAX_STAT_DATA_VERSIONS = 3; public static final boolean DEFAULT_META_DATA_KEEP_DELETED_CELLS = true; - + + // Min system table timestamps for every release. + public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0 = MIN_TABLE_TIMESTAMP + 3; + public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_2_0 = MIN_TABLE_TIMESTAMP + 4; + public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_2_1 = MIN_TABLE_TIMESTAMP + 5; + // TODO: pare this down to minimum, as we don't need duplicates for both table and column errors, nor should we need // a different code for every type of error. // ENTITY_ALREADY_EXISTS, ENTITY_NOT_FOUND, NEWER_ENTITY_FOUND, ENTITY_NOT_IN_REGION, CONCURRENT_MODIFICATION http://git-wip-us.apache.org/repos/asf/phoenix/blob/78d91d11/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index 4a9eac0..2b508b5 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -1847,20 +1847,20 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement // This will occur if we have an older SYSTEM.CATALOG and we need to update it to include // any new columns we've added. long currentServerSideTableTimeStamp = e.getTable().getTimeStamp(); - + // We know that we always need to add the STORE_NULLS column for 4.3 release String columnsToAdd = PhoenixDatabaseMetaData.STORE_NULLS + " " + PBoolean.INSTANCE.getSqlTypeName(); - - // If the server side schema is 4 versions behind then we need to add INDEX_TYPE - // and INDEX_DISABLE_TIMESTAMP columns too. + + // If the server side schema is at before MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0 then + // we need to add INDEX_TYPE and INDEX_DISABLE_TIMESTAMP columns too. // TODO: Once https://issues.apache.org/jira/browse/PHOENIX-1614 is fixed, // we should just have a ALTER TABLE ADD IF NOT EXISTS statement with all // the column names that have been added to SYSTEM.CATALOG since 4.0. - if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP - 4) { + if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0) { columnsToAdd += ", " + PhoenixDatabaseMetaData.INDEX_TYPE + " " + PUnsignedTinyint.INSTANCE.getSqlTypeName() + ", " + PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP + " " + PLong.INSTANCE.getSqlTypeName(); } - + // Ugh..need to assign to another local variable to keep eclipse happy. PhoenixConnection newMetaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, @@ -1876,14 +1876,23 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement } catch (NewerTableAlreadyExistsException e) { // Ignore, as this will happen if the SYSTEM.SEQUENCE already exists at this fixed timestamp. // A TableAlreadyExistsException is not thrown, since the table only exists *after* this fixed timestamp. - Integer sequenceSaltBuckets = e.getTable().getBucketNum(); - nSequenceSaltBuckets = sequenceSaltBuckets == null ? 0 : sequenceSaltBuckets; + nSequenceSaltBuckets = getSaltBuckets(e); } catch (TableAlreadyExistsException e) { // This will occur if we have an older SYSTEM.SEQUENCE and we need to update it to include // any new columns we've added. long currentServerSideTableTimeStamp = e.getTable().getTimeStamp(); - // if the table is at a timestamp corresponding to before 4.2.1 then run the upgrade script - if (currentServerSideTableTimeStamp <= MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP - 2) { + if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0) { + // If the table time stamp is before 4.1.0 then we need to add below columns + // to the SYSTEM.SEQUENCE table. + String columnsToAdd = PhoenixDatabaseMetaData.MIN_VALUE + " " + PLong.INSTANCE.getSqlTypeName() + + ", " + PhoenixDatabaseMetaData.MAX_VALUE + " " + PLong.INSTANCE.getSqlTypeName() + + ", " + PhoenixDatabaseMetaData.CYCLE_FLAG + " " + PBoolean.INSTANCE.getSqlTypeName() + + ", " + PhoenixDatabaseMetaData.LIMIT_REACHED_FLAG + " " + PBoolean.INSTANCE.getSqlTypeName(); + addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, + MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP, columnsToAdd); + } + // If the table timestamp is before 4.2.1 then run the upgrade script + if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_2_1) { if (UpgradeUtil.upgradeSequenceTable(metaConnection, nSaltBuckets, e.getTable())) { metaConnection.removeTable(null, PhoenixDatabaseMetaData.SEQUENCE_SCHEMA_NAME, @@ -1896,16 +1905,8 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement clearTableRegionCache(PhoenixDatabaseMetaData.SEQUENCE_FULLNAME_BYTES); } nSequenceSaltBuckets = nSaltBuckets; - } - if (currentServerSideTableTimeStamp <= MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP - 3) { - // If the table time stamp is before 4.1.0 then we need to add below columns - // to the SYSTEM.SEQUENCE table. - String columnsToAdd = PhoenixDatabaseMetaData.MIN_VALUE + " " + PLong.INSTANCE.getSqlTypeName() - + ", " + PhoenixDatabaseMetaData.MAX_VALUE + " " + PLong.INSTANCE.getSqlTypeName() - + ", " + PhoenixDatabaseMetaData.CYCLE_FLAG + " " + PBoolean.INSTANCE.getSqlTypeName() - + ", " + PhoenixDatabaseMetaData.LIMIT_REACHED_FLAG + " " + PBoolean.INSTANCE.getSqlTypeName(); - addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, - MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP, columnsToAdd); + } else { + nSequenceSaltBuckets = getSaltBuckets(e); } } try { @@ -1955,7 +1956,13 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement throw Throwables.propagate(e); } } - + + private static int getSaltBuckets(TableAlreadyExistsException e) { + PTable table = e.getTable(); + Integer sequenceSaltBuckets = table == null ? null : table.getBucketNum(); + return sequenceSaltBuckets == null ? 0 : sequenceSaltBuckets; + } + @Override public MutationState updateData(MutationPlan plan) throws SQLException { return plan.execute(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/78d91d11/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java index a92223b..86da5cc 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java @@ -220,14 +220,15 @@ public class UpgradeUtil { preSplitSequenceTable(conn, nSaltBuckets); return true; } - // We can detect upgrade from 4.2.0 -> 4.2.1 based on the timestamp of the table row - if (oldTable.getTimeStamp() == MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP-1) { + // If upgrading from 4.2.0, then we need this special case of pre-splitting the table. + // This is needed as a fix for https://issues.apache.org/jira/browse/PHOENIX-1401 + if (oldTable.getTimeStamp() == MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_2_0) { byte[] oldSeqNum = PLong.INSTANCE.toBytes(oldTable.getSequenceNumber()); KeyValue seqNumKV = KeyValueUtil.newKeyValue(seqTableKey, PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.TABLE_SEQ_NUM_BYTES, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP, - PLong.INSTANCE.toBytes(oldTable.getSequenceNumber()+1)); + PLong.INSTANCE.toBytes(MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP)); Put seqNumPut = new Put(seqTableKey); seqNumPut.add(seqNumKV); // Increment TABLE_SEQ_NUM in checkAndPut as semaphore so that only single client @@ -243,8 +244,9 @@ public class UpgradeUtil { return false; } - // if the SYSTEM.SEQUENCE table is for 4.1.0 or before then we need to salt the table - if (oldTable.getTimeStamp() <= MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP - 3) { + // if the SYSTEM.SEQUENCE table is at 4.1.0 or before then we need to salt the table + // and pre-split it. + if (oldTable.getTimeStamp() <= MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0) { int batchSizeBytes = 100 * 1024; // 100K chunks int sizeBytes = 0; List<Mutation> mutations = Lists.newArrayListWithExpectedSize(10000);
