http://git-wip-us.apache.org/repos/asf/phoenix/blob/e78eb6fa/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java index 3a0b03b..d1b3b27 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java @@ -67,7 +67,6 @@ import org.apache.phoenix.util.KeyValueUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.StringUtil; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -209,6 +208,8 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData, org.apache.pho public static final byte[] IS_VIEW_REFERENCED_BYTES = Bytes.toBytes(IS_VIEW_REFERENCED); public static final String VIEW_INDEX_ID = "VIEW_INDEX_ID"; public static final byte[] VIEW_INDEX_ID_BYTES = Bytes.toBytes(VIEW_INDEX_ID); + public static final String BASE_COLUMN_COUNT = "BASE_COLUMN_COUNT"; + public static final byte[] BASE_COLUMN_COUNT_BYTES = Bytes.toBytes(BASE_COLUMN_COUNT); public static final String TABLE_FAMILY = QueryConstants.DEFAULT_COLUMN_FAMILY; public static final byte[] TABLE_FAMILY_BYTES = QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e78eb6fa/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index d6d5df9..c5dde10 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -20,6 +20,7 @@ package org.apache.phoenix.query; import static org.apache.hadoop.hbase.HColumnDescriptor.TTL; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES; import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_DROP_METADATA; +import static org.apache.phoenix.util.UpgradeUtil.upgradeTo4_5_0; import java.io.IOException; import java.sql.SQLException; @@ -116,6 +117,7 @@ import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.ConnectionInfo; import org.apache.phoenix.parse.PFunction; import org.apache.phoenix.protobuf.ProtobufUtil; +import org.apache.phoenix.schema.ColumnAlreadyExistsException; import org.apache.phoenix.schema.ColumnFamilyNotFoundException; import org.apache.phoenix.schema.EmptySequenceCacheException; import org.apache.phoenix.schema.FunctionNotFoundException; @@ -141,6 +143,7 @@ import org.apache.phoenix.schema.stats.PTableStats; import org.apache.phoenix.schema.stats.StatisticsUtil; import org.apache.phoenix.schema.types.PBoolean; import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PInteger; import org.apache.phoenix.schema.types.PLong; import org.apache.phoenix.schema.types.PUnsignedTinyint; import org.apache.phoenix.util.ByteUtil; @@ -1823,21 +1826,16 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement } - /** - * Keeping this to use for further upgrades. This method closes the oldMetaConnection. - */ - private PhoenixConnection addColumnsIfNotExists(PhoenixConnection oldMetaConnection, - String tableName, long timestamp, String columns) throws SQLException { - + private PhoenixConnection addColumn(PhoenixConnection oldMetaConnection, String tableName, long timestamp, String columns, boolean addIfNotExists) throws SQLException { Properties props = new Properties(oldMetaConnection.getClientInfo()); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(timestamp)); // Cannot go through DriverManager or you end up in an infinite loop because it'll call init again PhoenixConnection metaConnection = new PhoenixConnection(this, oldMetaConnection.getURL(), props, oldMetaConnection.getMetaDataCache()); SQLException sqlE = null; try { - metaConnection.createStatement().executeUpdate("ALTER TABLE " + tableName + " ADD IF NOT EXISTS " + columns ); + metaConnection.createStatement().executeUpdate("ALTER TABLE " + tableName + " ADD " + (addIfNotExists ? " IF NOT EXISTS " : "") + columns ); } catch (SQLException e) { - logger.warn("addColumnsIfNotExists failed due to:" + e); + logger.warn("Add column failed due to:" + e); sqlE = e; } finally { try { @@ -1855,6 +1853,14 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement } return metaConnection; } + + /** + * Keeping this to use for further upgrades. This method closes the oldMetaConnection. + */ + private PhoenixConnection addColumnsIfNotExists(PhoenixConnection oldMetaConnection, + String tableName, long timestamp, String columns) throws SQLException { + return addColumn(oldMetaConnection, tableName, timestamp, columns, true); + } @Override public void init(final String url, final Properties props) throws SQLException { @@ -1926,21 +1932,36 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement } } - // If the server side schema is at before MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0 then - // we need to add INDEX_TYPE and INDEX_DISABLE_TIMESTAMP columns too. - // TODO: Once https://issues.apache.org/jira/browse/PHOENIX-1614 is fixed, - // we should just have a ALTER TABLE ADD IF NOT EXISTS statement with all - // the column names that have been added to SYSTEM.CATALOG since 4.0. + // If the server side schema is before MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0 then + // we need to add INDEX_TYPE and INDEX_DISABLE_TIMESTAMP columns too. + // TODO: Once https://issues.apache.org/jira/browse/PHOENIX-1614 is fixed, + // we should just have a ALTER TABLE ADD IF NOT EXISTS statement with all + // the column names that have been added to SYSTEM.CATALOG since 4.0. if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0) { columnsToAdd += ", " + PhoenixDatabaseMetaData.INDEX_TYPE + " " + PUnsignedTinyint.INSTANCE.getSqlTypeName() + ", " + PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP + " " + PLong.INSTANCE.getSqlTypeName(); } - // Ugh..need to assign to another local variable to keep eclipse happy. PhoenixConnection newMetaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP, columnsToAdd); metaConnection = newMetaConnection; + + if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_5_0) { + columnsToAdd = PhoenixDatabaseMetaData.BASE_COLUMN_COUNT + " " + + PInteger.INSTANCE.getSqlTypeName(); + try { + addColumn(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, + MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP, columnsToAdd, false); + upgradeTo4_5_0(metaConnection); + } catch (ColumnAlreadyExistsException ignored) { + /* + * Upgrade to 4.5 is a slightly special case. We use the fact that the column + * BASE_COLUMN_COUNT is already part of the meta-data schema as the signal that + * the server side upgrade has finished or is in progress. + */ + } + } } int nSaltBuckets = ConnectionQueryServicesImpl.this.props.getInt(QueryServices.SEQUENCE_SALT_BUCKETS_ATTRIB, QueryServicesOptions.DEFAULT_SEQUENCE_TABLE_SALT_BUCKETS); @@ -1983,6 +2004,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement } else { nSequenceSaltBuckets = getSaltBuckets(e); } + } try { metaConnection.createStatement().executeUpdate( @@ -2002,6 +2024,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement } catch (NewerTableAlreadyExistsException e) { } catch (TableAlreadyExistsException e) { } + } catch (Exception e) { if (e instanceof SQLException) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/e78eb6fa/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java index 73d1123..f82c594 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java @@ -18,10 +18,13 @@ package org.apache.phoenix.query; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARG_POSITION; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARRAY_SIZE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.BASE_COLUMN_COUNT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.BUFFER_LENGTH; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CACHE_SIZE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CHAR_OCTET_LENGTH; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLASS_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_COUNT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_DEF; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_FAMILY; @@ -33,7 +36,9 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TABLE_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DECIMAL_DIGITS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_VALUE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DISABLE_WAL; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.FUNCTION_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_COUNT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT; @@ -43,9 +48,12 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INCREMENT_BY; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_STATE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_TYPE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_ARRAY; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_AUTOINCREMENT; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_CONSTANT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_NULLABLE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_VIEW_REFERENCED; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.JAR_PATH; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.KEY_SEQ; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LAST_STATS_UPDATE_TIME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LIMIT_REACHED_FLAG; @@ -56,6 +64,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MIN_KEY; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MIN_VALUE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MULTI_TENANT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NULLABLE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NUM_ARGS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NUM_PREC_RADIX; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ORDINAL_POSITION; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PHYSICAL_NAME; @@ -63,7 +72,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PK_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.REF_GENERATION; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.REGION_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.REMARKS; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.STORE_NULLS; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.RETURN_TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SALT_BUCKETS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCOPE_CATALOG; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCOPE_SCHEMA; @@ -76,27 +85,19 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SOURCE_DATA_TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SQL_DATA_TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SQL_DATETIME_SUB; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.START_WITH; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.STORE_NULLS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_FUNCTION_TABLE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_SEQUENCE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLASS_NAME; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_FUNCTION_TABLE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARG_POSITION; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.FUNCTION_NAME; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.JAR_PATH; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NUM_ARGS; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_ARRAY; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_CONSTANT; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_VALUE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.RETURN_TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT; @@ -184,6 +185,8 @@ public interface QueryConstants { public static final BigDecimal BD_MILLIS_IN_DAY = BigDecimal.valueOf(QueryConstants.MILLIS_IN_DAY); public static final int MAX_ALLOWED_NANOS = 999999999; public static final int NANOS_IN_SECOND = BigDecimal.valueOf(Math.pow(10, 9)).intValue(); + public static final int DIVORCED_VIEW_BASE_COLUMN_COUNT = -100; + public static final int BASE_TABLE_BASE_COLUMN_COUNT = -1; public static final String CREATE_TABLE_METADATA = // Do not use IF NOT EXISTS as we sometimes catch the TableAlreadyExists // exception and add columns to the SYSTEM.TABLE dynamically. @@ -241,7 +244,8 @@ public interface QueryConstants { IS_AUTOINCREMENT + " VARCHAR," + INDEX_TYPE + " UNSIGNED_TINYINT," + INDEX_DISABLE_TIMESTAMP + " BIGINT," + - STORE_NULLS + " BOOLEAN," + + STORE_NULLS + " BOOLEAN," + + BASE_COLUMN_COUNT + " INTEGER," + "CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + "," + TABLE_SCHEM + "," + TABLE_NAME + "," + COLUMN_NAME + "," + COLUMN_FAMILY + "))\n" + HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" + http://git-wip-us.apache.org/repos/asf/phoenix/blob/e78eb6fa/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java index b719aae..2a43679 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java @@ -236,4 +236,9 @@ public class DelegateTable implements PTable { public PName getParentSchemaName() { return delegate.getParentSchemaName(); } + + @Override + public int getBaseColumnCount() { + return delegate.getBaseColumnCount(); + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/e78eb6fa/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 75678fd..e7c3cd5 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -22,7 +22,10 @@ import static com.google.common.collect.Sets.newLinkedHashSet; import static com.google.common.collect.Sets.newLinkedHashSetWithExpectedSize; import static org.apache.hadoop.hbase.HColumnDescriptor.TTL; import static org.apache.phoenix.exception.SQLExceptionCode.INSUFFICIENT_MULTI_TENANT_COLUMNS; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARG_POSITION; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARRAY_SIZE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.BASE_COLUMN_COUNT; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLASS_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_COUNT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_DEF; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_FAMILY; @@ -34,6 +37,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DECIMAL_DIGITS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_VALUE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DISABLE_WAL; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.FUNCTION_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_STATE; @@ -41,6 +45,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_ARRAY; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_CONSTANT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_VIEW_REFERENCED; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.JAR_PATH; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.KEY_SEQ; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LAST_STATS_UPDATE_TIME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LINK_TYPE; @@ -48,39 +53,34 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MAX_VALUE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MIN_VALUE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MULTI_TENANT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NULLABLE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NUM_ARGS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ORDINAL_POSITION; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PARENT_TENANT_ID; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PHYSICAL_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PK_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.REGION_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.RETURN_TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SALT_BUCKETS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SORT_ORDER; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.STORE_NULLS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_FUNCTION_TABLE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARG_POSITION; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NUM_ARGS; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_FUNCTION_TABLE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.FUNCTION_NAME; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLASS_NAME; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.JAR_PATH; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.RETURN_TYPE; +import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT; import static org.apache.phoenix.query.QueryServices.DROP_METADATA_ATTRIB; import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_DROP_METADATA; import java.io.IOException; -import java.lang.reflect.Constructor; -import java.sql.Array; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ParameterMetaData; @@ -115,7 +115,6 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.io.TimeRange; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.DynamicClassLoader; import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.compile.ColumnResolver; import org.apache.phoenix.compile.ExplainPlan; @@ -137,8 +136,6 @@ import org.apache.phoenix.execute.MutationState; import org.apache.phoenix.expression.Determinism; import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.RowKeyColumnExpression; -import org.apache.phoenix.expression.function.FunctionExpression; -import org.apache.phoenix.expression.function.ScalarFunction; import org.apache.phoenix.hbase.index.covered.update.ColumnReference; import org.apache.phoenix.index.IndexMaintainer; import org.apache.phoenix.jdbc.PhoenixConnection; @@ -158,18 +155,16 @@ import org.apache.phoenix.parse.DropFunctionStatement; import org.apache.phoenix.parse.DropIndexStatement; import org.apache.phoenix.parse.DropSequenceStatement; import org.apache.phoenix.parse.DropTableStatement; -import org.apache.phoenix.parse.FunctionParseNode; import org.apache.phoenix.parse.IndexKeyConstraint; import org.apache.phoenix.parse.NamedTableNode; import org.apache.phoenix.parse.PFunction; +import org.apache.phoenix.parse.PFunction.FunctionArgument; import org.apache.phoenix.parse.ParseNode; import org.apache.phoenix.parse.ParseNodeFactory; import org.apache.phoenix.parse.PrimaryKeyConstraint; import org.apache.phoenix.parse.TableName; import org.apache.phoenix.parse.UpdateStatisticsStatement; -import org.apache.phoenix.parse.PFunction.FunctionArgument; import org.apache.phoenix.query.ConnectionQueryServices.Feature; -import org.apache.phoenix.query.HBaseFactoryProvider; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; @@ -225,8 +220,9 @@ public class MetaDataClient { VIEW_TYPE + "," + VIEW_INDEX_ID + "," + INDEX_TYPE + "," + - STORE_NULLS + - ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + STORE_NULLS + "," + + BASE_COLUMN_COUNT + + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; private static final String CREATE_LINK = "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " + TENANT_ID + "," + @@ -1878,6 +1874,11 @@ public class MetaDataClient { tableUpsert.setByte(18, indexType.getSerializedValue()); } tableUpsert.setBoolean(19, storeNulls); + if (parent != null && tableType == PTableType.VIEW) { + tableUpsert.setInt(20, parent.getColumns().size()); + } else { + tableUpsert.setInt(20, BASE_TABLE_BASE_COLUMN_COUNT); + } tableUpsert.execute(); tableMetaData.addAll(connection.getMutationState().toMutations().next().getSecond()); http://git-wip-us.apache.org/repos/asf/phoenix/blob/e78eb6fa/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java index e46dcb7..b983074 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java @@ -322,4 +322,5 @@ public interface PTable extends PMetaDataEntity { IndexType getIndexType(); PTableStats getTableStats(); + int getBaseColumnCount(); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/e78eb6fa/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java index b62dbf5..25b3e87 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java @@ -125,6 +125,7 @@ public class PTableImpl implements PTable { private int estimatedSize; private IndexType indexType; private PTableStats tableStats = PTableStats.EMPTY_STATS; + private int baseColumnCount; public PTableImpl() { this.indexes = Collections.emptyList(); @@ -193,7 +194,7 @@ public class PTableImpl implements PTable { table.getTenantId(), table.getSchemaName(), table.getTableName(), table.getType(), table.getIndexState(), timeStamp, table.getSequenceNumber(), table.getPKName(), table.getBucketNum(), getColumnsToClone(table), parentSchemaName, table.getParentTableName(), indexes, table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), viewStatement, - table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats()); + table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats(), table.getBaseColumnCount()); } public static PTableImpl makePTable(PTable table, List<PColumn> columns) throws SQLException { @@ -201,7 +202,7 @@ public class PTableImpl implements PTable { table.getTenantId(), table.getSchemaName(), table.getTableName(), table.getType(), table.getIndexState(), table.getTimeStamp(), table.getSequenceNumber(), table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), - table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats()); + table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats(), table.getBaseColumnCount()); } public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, List<PColumn> columns) throws SQLException { @@ -209,7 +210,7 @@ public class PTableImpl implements PTable { table.getTenantId(), table.getSchemaName(), table.getTableName(), table.getType(), table.getIndexState(), timeStamp, sequenceNumber, table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(), - table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats()); + table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats(), table.getBaseColumnCount()); } public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, List<PColumn> columns, boolean isImmutableRows) throws SQLException { @@ -217,7 +218,7 @@ public class PTableImpl implements PTable { table.getTenantId(), table.getSchemaName(), table.getTableName(), table.getType(), table.getIndexState(), timeStamp, sequenceNumber, table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(), isImmutableRows, table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), - table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats()); + table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats(), table.getBaseColumnCount()); } public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, List<PColumn> columns, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls) throws SQLException { @@ -225,7 +226,7 @@ public class PTableImpl implements PTable { table.getTenantId(), table.getSchemaName(), table.getTableName(), table.getType(), table.getIndexState(), timeStamp, sequenceNumber, table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(), isImmutableRows, table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), - isWalDisabled, isMultitenant, storeNulls, table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats()); + isWalDisabled, isMultitenant, storeNulls, table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats(), table.getBaseColumnCount()); } public static PTableImpl makePTable(PTable table, PIndexState state) throws SQLException { @@ -234,7 +235,7 @@ public class PTableImpl implements PTable { table.getSequenceNumber(), table.getPKName(), table.getBucketNum(), getColumnsToClone(table), table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), - table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats()); + table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats(), table.getBaseColumnCount()); } public static PTableImpl makePTable(PTable table, PTableStats stats) throws SQLException { @@ -243,7 +244,7 @@ public class PTableImpl implements PTable { table.getSequenceNumber(), table.getPKName(), table.getBucketNum(), getColumnsToClone(table), table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), - table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), stats); + table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), stats, table.getBaseColumnCount()); } public static PTableImpl makePTable(PName tenantId, PName schemaName, PName tableName, PTableType type, PIndexState state, long timeStamp, long sequenceNumber, @@ -253,18 +254,18 @@ public class PTableImpl implements PTable { return new PTableImpl(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName, bucketNum, columns, dataSchemaName, dataTableName, indexes, isImmutableRows, physicalNames, defaultFamilyName, viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, - indexType, PTableStats.EMPTY_STATS); + indexType, PTableStats.EMPTY_STATS, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT); } public static PTableImpl makePTable(PName tenantId, PName schemaName, PName tableName, PTableType type, PIndexState state, long timeStamp, long sequenceNumber, PName pkName, Integer bucketNum, List<PColumn> columns, PName dataSchemaName, PName dataTableName, List<PTable> indexes, boolean isImmutableRows, List<PName> physicalNames, PName defaultFamilyName, String viewExpression, - boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType, @NotNull PTableStats stats) + boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType, @NotNull PTableStats stats, int baseColumnCount) throws SQLException { return new PTableImpl(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName, bucketNum, columns, dataSchemaName, dataTableName, indexes, isImmutableRows, physicalNames, - defaultFamilyName, viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, stats); + defaultFamilyName, viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, stats, baseColumnCount); } private PTableImpl(PName tenantId, PName schemaName, PName tableName, PTableType type, PIndexState state, @@ -272,10 +273,10 @@ public class PTableImpl implements PTable { PName parentSchemaName, PName parentTableName, List<PTable> indexes, boolean isImmutableRows, List<PName> physicalNames, PName defaultFamilyName, String viewExpression, boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType, - PTableStats stats) throws SQLException { + PTableStats stats, int baseColumnCount) throws SQLException { init(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName, bucketNum, columns, stats, schemaName, parentTableName, indexes, isImmutableRows, physicalNames, defaultFamilyName, - viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType); + viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount); } @Override @@ -303,7 +304,7 @@ public class PTableImpl implements PTable { PName pkName, Integer bucketNum, List<PColumn> columns, PTableStats stats, PName parentSchemaName, PName parentTableName, List<PTable> indexes, boolean isImmutableRows, List<PName> physicalNames, PName defaultFamilyName, String viewExpression, boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId, - IndexType indexType ) throws SQLException { + IndexType indexType , int baseColumnCount) throws SQLException { Preconditions.checkNotNull(schemaName); Preconditions.checkArgument(tenantId==null || tenantId.getBytes().length > 0); // tenantId should be null or not empty int estimatedSize = SizedUtil.OBJECT_SIZE * 2 + 23 * SizedUtil.POINTER_SIZE + 4 * SizedUtil.INT_SIZE + 2 * SizedUtil.LONG_SIZE + 2 * SizedUtil.INT_OBJECT_SIZE + @@ -437,6 +438,7 @@ public class PTableImpl implements PTable { } this.estimatedSize = estimatedSize; + this.baseColumnCount = baseColumnCount; } @Override @@ -968,13 +970,18 @@ public class PTableImpl implements PTable { physicalNames.add(PNameFactory.newName(table.getPhysicalNames(i).toByteArray())); } } + + int baseColumnCount = -1; + if (table.hasBaseColumnCount()) { + baseColumnCount = table.getBaseColumnCount(); + } try { PTableImpl result = new PTableImpl(); result.init(tenantId, schemaName, tableName, tableType, indexState, timeStamp, sequenceNumber, pkName, (bucketNum == NO_SALTING) ? null : bucketNum, columns, stats, schemaName,dataTableName, indexes, isImmutableRows, physicalNames, defaultFamilyName, viewStatement, disableWAL, - multiTenant, storeNulls, viewType, viewIndexId, indexType); + multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount); return result; } catch (SQLException e) { throw new RuntimeException(e); // Impossible @@ -1063,6 +1070,7 @@ public class PTableImpl implements PTable { builder.addPhysicalNames(ByteStringer.wrap(table.getPhysicalNames().get(i).getBytes())); } } + builder.setBaseColumnCount(table.getBaseColumnCount()); return builder.build(); } @@ -1082,4 +1090,8 @@ public class PTableImpl implements PTable { return parentSchemaName; } + @Override + public int getBaseColumnCount() { + return baseColumnCount; + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/e78eb6fa/phoenix-core/src/main/java/org/apache/phoenix/util/ByteUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ByteUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ByteUtil.java index 1e3516d..1f4a285 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/ByteUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ByteUtil.java @@ -253,13 +253,17 @@ public class ByteUtil { public static byte[] concat(byte[] first, byte[]... rest) { int totalLength = first.length; for (byte[] array : rest) { - totalLength += array.length; + if (array != null) { + totalLength += array.length; + } } byte[] result = Arrays.copyOf(first, totalLength); int offset = first.length; for (byte[] array : rest) { - System.arraycopy(array, 0, result, offset, array.length); - offset += array.length; + if (array != null) { + System.arraycopy(array, 0, result, offset, array.length); + offset += array.length; + } } return result; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/e78eb6fa/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java index 86da5cc..dff6598 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java @@ -17,9 +17,35 @@ */ package org.apache.phoenix.util; +import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARRAY_SIZE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_FAMILY; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_SIZE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TYPE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DECIMAL_DIGITS; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LINK_TYPE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ORDINAL_POSITION; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SORT_ORDER; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID; +import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT; +import static org.apache.phoenix.query.QueryConstants.DIVORCED_VIEW_BASE_COLUMN_COUNT; + import java.io.IOException; +import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; @@ -38,21 +64,43 @@ import org.apache.phoenix.coprocessor.MetaDataProtocol; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.query.QueryConstants; -import org.apache.phoenix.schema.types.PInteger; -import org.apache.phoenix.schema.types.PLong; import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PNameFactory; import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTable.LinkType; import org.apache.phoenix.schema.SaltingUtil; +import org.apache.phoenix.schema.types.PInteger; +import org.apache.phoenix.schema.types.PLong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Objects; import com.google.common.collect.Lists; public class UpgradeUtil { private static final Logger logger = LoggerFactory.getLogger(UpgradeUtil.class); private static final byte[] SEQ_PREFIX_BYTES = ByteUtil.concat(QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes("_SEQ_")); + + public static String UPSERT_BASE_COLUMN_COUNT_IN_HEADER_ROW = "UPSERT " + + "INTO SYSTEM.CATALOG " + + "(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY, BASE_COLUMN_COUNT) " + + "VALUES (?, ?, ?, ?, ?, ?) "; + public static String SELECT_BASE_COLUMN_COUNT_FROM_HEADER_ROW = "SELECT " + + "BASE_COLUMN_COUNT " + + "FROM SYSTEM.CATALOG " + + "WHERE " + + "COLUMN_NAME IS NULL " + + "AND " + + "COLUMN_FAMILY IS NULL " + + "AND " + + "TENANT_ID %s " + + "AND " + + "TABLE_SCHEM %s " + + "AND " + + "TABLE_NAME = ? " + ; + private UpgradeUtil() { } @@ -394,5 +442,348 @@ public class UpgradeUtil { keyValue.getTimestamp(), KeyValue.Type.codeToType(keyValue.getType()), buf, keyValue.getValueOffset(), keyValue.getValueLength()); } + + public static void upgradeTo4_5_0(PhoenixConnection metaConnection) throws SQLException { + String getBaseTableAndViews = "SELECT " + + COLUMN_FAMILY + " AS BASE_PHYSICAL_TABLE, " + + TENANT_ID + ", " + + TABLE_SCHEM + " AS VIEW_SCHEMA, " + + TABLE_NAME + " AS VIEW_NAME " + + "FROM " + SYSTEM_CATALOG_NAME + + " WHERE " + COLUMN_FAMILY + " IS NOT NULL " // column_family column points to the physical table name. + + " AND " + COLUMN_NAME + " IS NULL " + + " AND " + LINK_TYPE + " = ? "; + // Build a map of base table name -> list of views on the table. + Map<String, List<ViewKey>> parentTableViewsMap = new HashMap<>(); + try (PreparedStatement stmt = metaConnection.prepareStatement(getBaseTableAndViews)) { + // Get back view rows that have links back to the base physical table. This takes care + // of cases when we have a hierarchy of views too. + stmt.setByte(1, LinkType.PHYSICAL_TABLE.getSerializedValue()); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + // this is actually SCHEMANAME.TABLENAME + String parentTable = rs.getString("BASE_PHYSICAL_TABLE"); + String tenantId = rs.getString(TENANT_ID); + String viewSchema = rs.getString("VIEW_SCHEMA"); + String viewName = rs.getString("VIEW_NAME"); + List<ViewKey> viewKeysList = parentTableViewsMap.get(parentTable); + if (viewKeysList == null) { + viewKeysList = new ArrayList<>(); + parentTableViewsMap.put(parentTable, viewKeysList); + } + viewKeysList.add(new ViewKey(tenantId, viewSchema, viewName)); + } + } + } + + for (Entry<String, List<ViewKey>> entry : parentTableViewsMap.entrySet()) { + // Fetch column information for the base physical table + String physicalTable = entry.getKey(); + String baseTableSchemaName = SchemaUtil.getSchemaNameFromFullName(physicalTable).equals(StringUtil.EMPTY_STRING) ? null : SchemaUtil.getSchemaNameFromFullName(physicalTable); + String baseTableName = SchemaUtil.getTableNameFromFullName(physicalTable); + List<ColumnDetails> basePhysicalTableColumns = new ArrayList<>(); + + // Columns fetched in order of ordinal position + String fetchColumnInfoForBasePhysicalTable = "SELECT " + + COLUMN_NAME + "," + + COLUMN_FAMILY + "," + + DATA_TYPE + "," + + COLUMN_SIZE + "," + + DECIMAL_DIGITS + "," + + ORDINAL_POSITION + "," + + SORT_ORDER + "," + + ARRAY_SIZE + " " + + "FROM SYSTEM.CATALOG " + + "WHERE " + + "TABLE_SCHEM %s " + + "AND TABLE_NAME = ? " + + "AND COLUMN_NAME IS NOT NULL " + + "ORDER BY " + + ORDINAL_POSITION; + + PreparedStatement stmt = null; + if (baseTableSchemaName == null) { + fetchColumnInfoForBasePhysicalTable = + String.format(fetchColumnInfoForBasePhysicalTable, "IS NULL "); + stmt = metaConnection.prepareStatement(fetchColumnInfoForBasePhysicalTable); + stmt.setString(1, baseTableName); + } else { + fetchColumnInfoForBasePhysicalTable = + String.format(fetchColumnInfoForBasePhysicalTable, " = ? "); + stmt = metaConnection.prepareStatement(fetchColumnInfoForBasePhysicalTable); + stmt.setString(1, baseTableSchemaName); + stmt.setString(2, baseTableName); + } + + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + basePhysicalTableColumns.add(new ColumnDetails(rs.getString(COLUMN_FAMILY), rs + .getString(COLUMN_NAME), rs.getInt(ORDINAL_POSITION), rs + .getInt(DATA_TYPE), rs.getInt(COLUMN_SIZE), rs.getInt(DECIMAL_DIGITS), + rs.getInt(SORT_ORDER), rs.getInt(ARRAY_SIZE))); + } + } + + // Fetch column information for all the views on the base physical table ordered by ordinal position. + List<ViewKey> viewKeys = entry.getValue(); + StringBuilder sb = new StringBuilder(); + sb.append("SELECT " + + TENANT_ID + "," + + TABLE_SCHEM + "," + + TABLE_NAME + "," + + COLUMN_NAME + "," + + COLUMN_FAMILY + "," + + DATA_TYPE + "," + + COLUMN_SIZE + "," + + DECIMAL_DIGITS + "," + + ORDINAL_POSITION + "," + + SORT_ORDER + "," + + ARRAY_SIZE + " " + + "FROM SYSTEM.CATALOG " + + "WHERE " + + COLUMN_NAME + " IS NOT NULL " + + "AND " + + ORDINAL_POSITION + " <= ? " + // fetch only those columns that would impact setting of base column count + "AND " + + "(" + TENANT_ID+ ", " + TABLE_SCHEM + ", " + TABLE_NAME + ") IN ("); + + int numViews = viewKeys.size(); + for (int i = 0; i < numViews; i++) { + sb.append(" (?, ?, ?) "); + if (i < numViews - 1) { + sb.append(", "); + } + } + sb.append(" ) "); + sb.append(" GROUP BY " + + TENANT_ID + "," + + TABLE_SCHEM + "," + + TABLE_NAME + "," + + COLUMN_NAME + "," + + COLUMN_FAMILY + "," + + DATA_TYPE + "," + + COLUMN_SIZE + "," + + DECIMAL_DIGITS + "," + + ORDINAL_POSITION + "," + + SORT_ORDER + "," + + ARRAY_SIZE + " " + + "ORDER BY " + + TENANT_ID + "," + TABLE_SCHEM + ", " + TABLE_NAME + ", " + ORDINAL_POSITION); + String fetchViewColumnsSql = sb.toString(); + stmt = metaConnection.prepareStatement(fetchViewColumnsSql); + int numColsInBaseTable = basePhysicalTableColumns.size(); + stmt.setInt(1, numColsInBaseTable); + int paramIndex = 1; + stmt.setInt(paramIndex++, numColsInBaseTable); + for (ViewKey view : viewKeys) { + stmt.setString(paramIndex++, view.tenantId); + stmt.setString(paramIndex++, view.schema); + stmt.setString(paramIndex++, view.name); + } + String currentTenantId = null; + String currentViewSchema = null; + String currentViewName = null; + try (ResultSet rs = stmt.executeQuery()) { + int numBaseTableColsMatched = 0; + boolean ignore = false; + boolean baseColumnCountUpserted = false; + while (rs.next()) { + String viewTenantId = rs.getString(TENANT_ID); + String viewSchema = rs.getString(TABLE_SCHEM); + String viewName = rs.getString(TABLE_NAME); + if (!(Objects.equal(viewTenantId, currentTenantId) && Objects.equal(viewSchema, currentViewSchema) && Objects.equal(viewName, currentViewName))) { + // We are about to iterate through columns of a different view. Check whether base column count was upserted. + // If it wasn't then it is likely the case that a column inherited from the base table was dropped from view. + if (currentViewName != null && !baseColumnCountUpserted && numBaseTableColsMatched < numColsInBaseTable) { + upsertBaseColumnCountInHeaderRow(metaConnection, currentTenantId, currentViewSchema, currentViewName, DIVORCED_VIEW_BASE_COLUMN_COUNT); + } + // reset the values as we are now going to iterate over columns of a new view. + numBaseTableColsMatched = 0; + currentTenantId = viewTenantId; + currentViewSchema = viewSchema; + currentViewName = viewName; + ignore = false; + baseColumnCountUpserted = false; + } + if (!ignore) { + /* + * Iterate over all the columns of the base physical table and the columns of the view. Compare the + * two till one of the following happens: + * + * 1) We run into a view column which is different from column in the base physical table. + * This means that the view has divorced itself from the base physical table. In such a case + * we will set a special value for the base column count. That special value will also be used + * on the server side to filter out the divorced view so that meta-data changes on the base + * physical table are not propagated to it. + * + * 2) Every physical table column is present in the view. In that case we set the base column count + * as the number of columns in the base physical table. At that point we ignore rest of the columns + * of the view. + * + */ + ColumnDetails baseTableColumn = basePhysicalTableColumns.get(numBaseTableColsMatched); + String columName = rs.getString(COLUMN_NAME); + String columnFamily = rs.getString(COLUMN_FAMILY); + int ordinalPos = rs.getInt(ORDINAL_POSITION); + int dataType = rs.getInt(DATA_TYPE); + int columnSize = rs.getInt(COLUMN_SIZE); + int decimalDigits = rs.getInt(DECIMAL_DIGITS); + int sortOrder = rs.getInt(SORT_ORDER); + int arraySize = rs.getInt(ARRAY_SIZE); + ColumnDetails viewColumn = new ColumnDetails(columnFamily, columName, ordinalPos, dataType, columnSize, decimalDigits, sortOrder, arraySize); + if (baseTableColumn.equals(viewColumn)) { + numBaseTableColsMatched++; + if (numBaseTableColsMatched == numColsInBaseTable) { + upsertBaseColumnCountInHeaderRow(metaConnection, viewTenantId, viewSchema, viewName, numColsInBaseTable); + // No need to ignore the rest of the columns of the view here since the + // query retrieved only those columns that had ordinal position <= numColsInBaseTable + baseColumnCountUpserted = true; + } + } else { + // special value to denote that the view has divorced itself from the base physical table. + upsertBaseColumnCountInHeaderRow(metaConnection, viewTenantId, viewSchema, viewName, DIVORCED_VIEW_BASE_COLUMN_COUNT); + baseColumnCountUpserted = true; + // ignore rest of the rows for the view. + ignore = true; + } + } + } + } + // set base column count for the header row of the base table too. We use this information + // to figure out whether the upgrade is in progress or hasn't started. + upsertBaseColumnCountInHeaderRow(metaConnection, null, baseTableSchemaName, baseTableName, BASE_TABLE_BASE_COLUMN_COUNT); + metaConnection.commit(); + } + } + + private static void upsertBaseColumnCountInHeaderRow(PhoenixConnection metaConnection, + String tenantId, String schemaName, String viewOrTableName, int baseColumnCount) + throws SQLException { + try (PreparedStatement stmt = + metaConnection.prepareStatement(UPSERT_BASE_COLUMN_COUNT_IN_HEADER_ROW)) { + stmt.setString(1, tenantId); + stmt.setString(2, schemaName); + stmt.setString(3, viewOrTableName); + stmt.setString(4, null); + stmt.setString(5, null); + stmt.setInt(6, baseColumnCount); + stmt.executeUpdate(); + } + } + + private static class ColumnDetails { + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + columnName.hashCode(); + result = prime * result + ((columnFamily == null) ? 0 : columnFamily.hashCode()); + result = prime * result + arraySize; + result = prime * result + dataType; + result = prime * result + maxLength; + result = prime * result + ordinalValue; + result = prime * result + scale; + result = prime * result + sortOrder; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + ColumnDetails other = (ColumnDetails) obj; + if (!columnName.equals(other.columnName)) return false; + if (columnFamily == null) { + if (other.columnFamily != null) return false; + } else if (!columnFamily.equals(other.columnFamily)) return false; + if (arraySize != other.arraySize) return false; + if (dataType != other.dataType) return false; + if (maxLength != other.maxLength) return false; + if (ordinalValue != other.ordinalValue) return false; + if (scale != other.scale) return false; + if (sortOrder != other.sortOrder) return false; + return true; + } + + @Nullable + private final String columnFamily; + + @Nonnull + private final String columnName; + + private final int ordinalValue; + + private final int dataType; + + private final int maxLength; + + private final int scale; + + private final int sortOrder; + + private final int arraySize; + + ColumnDetails(String columnFamily, String columnName, int ordinalValue, int dataType, + int maxLength, int scale, int sortOrder, int arraySize) { + checkNotNull(columnName); + checkNotNull(ordinalValue); + checkNotNull(dataType); + this.columnFamily = columnFamily; + this.columnName = columnName; + this.ordinalValue = ordinalValue; + this.dataType = dataType; + this.maxLength = maxLength; + this.scale = scale; + this.sortOrder = sortOrder; + this.arraySize = arraySize; + } + + } + + private static class ViewKey { + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((tenantId == null) ? 0 : tenantId.hashCode()); + result = prime * result + name.hashCode(); + result = prime * result + ((schema == null) ? 0 : schema.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + ViewKey other = (ViewKey) obj; + if (tenantId == null) { + if (other.tenantId != null) return false; + } else if (!tenantId.equals(other.tenantId)) return false; + if (!name.equals(other.name)) return false; + if (schema == null) { + if (other.schema != null) return false; + } else if (!schema.equals(other.schema)) return false; + return true; + } + + @Nullable + private final String tenantId; + + @Nullable + private final String schema; + + @Nonnull + private final String name; + + private ViewKey(String tenantId, String schema, String viewName) { + this.tenantId = tenantId; + this.schema = schema; + this.name = viewName; + } + } }