PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d6a82e29 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d6a82e29 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d6a82e29 Branch: refs/heads/4.x-HBase-1.1 Commit: d6a82e29fce471bd47c42802ab4ac9548acf60ae Parents: 03c2bc2 Author: Samarth <samarth.j...@salesforce.com> Authored: Sun Feb 26 22:22:03 2017 -0800 Committer: Samarth <samarth.j...@salesforce.com> Committed: Sun Feb 26 22:22:03 2017 -0800 ---------------------------------------------------------------------- .../phoenix/end2end/AggregateQueryIT.java | 74 +- .../AlterMultiTenantTableWithViewsIT.java | 25 +- .../apache/phoenix/end2end/AlterTableIT.java | 491 +- .../phoenix/end2end/AlterTableWithViewsIT.java | 133 +- .../org/apache/phoenix/end2end/ArrayIT.java | 28 + .../org/apache/phoenix/end2end/BaseJoinIT.java | 4 +- .../org/apache/phoenix/end2end/BaseQueryIT.java | 99 +- .../apache/phoenix/end2end/CaseStatementIT.java | 28 +- .../apache/phoenix/end2end/CastAndCoerceIT.java | 34 +- .../end2end/ClientTimeArithmeticQueryIT.java | 76 +- .../end2end/ColumnEncodedBytesPropIT.java | 95 + .../end2end/CountDistinctCompressionIT.java | 2 +- .../apache/phoenix/end2end/CreateTableIT.java | 57 + .../org/apache/phoenix/end2end/DateTimeIT.java | 2 +- .../phoenix/end2end/DefaultColumnValueIT.java | 1 + .../apache/phoenix/end2end/DerivedTableIT.java | 2 +- .../apache/phoenix/end2end/DistinctCountIT.java | 4 +- .../apache/phoenix/end2end/DynamicColumnIT.java | 63 + .../phoenix/end2end/ExtendedQueryExecIT.java | 8 +- .../apache/phoenix/end2end/FunkyNamesIT.java | 2 +- .../org/apache/phoenix/end2end/GroupByIT.java | 162 +- .../phoenix/end2end/ImmutableTablePropIT.java | 130 - .../end2end/ImmutableTablePropertiesIT.java | 189 + .../apache/phoenix/end2end/MutableQueryIT.java | 424 ++ .../phoenix/end2end/NativeHBaseTypesIT.java | 2 +- .../org/apache/phoenix/end2end/NotQueryIT.java | 28 +- .../org/apache/phoenix/end2end/OrderByIT.java | 2 - .../apache/phoenix/end2end/PercentileIT.java | 4 +- .../phoenix/end2end/PhoenixRuntimeIT.java | 4 +- .../phoenix/end2end/PointInTimeQueryIT.java | 78 +- .../phoenix/end2end/ProductMetricsIT.java | 2 +- .../end2end/QueryDatabaseMetaDataIT.java | 16 +- .../org/apache/phoenix/end2end/QueryIT.java | 112 +- .../phoenix/end2end/ReadIsolationLevelIT.java | 2 +- .../phoenix/end2end/RowValueConstructorIT.java | 36 +- .../org/apache/phoenix/end2end/ScanQueryIT.java | 93 +- .../phoenix/end2end/StatsCollectorIT.java | 124 +- .../apache/phoenix/end2end/StoreNullsIT.java | 310 +- .../phoenix/end2end/StoreNullsPropIT.java | 51 + ...SysTableNamespaceMappedStatsCollectorIT.java | 4 +- .../java/org/apache/phoenix/end2end/TopNIT.java | 6 +- .../apache/phoenix/end2end/UpsertSelectIT.java | 10 +- .../apache/phoenix/end2end/UpsertValuesIT.java | 51 +- .../phoenix/end2end/UserDefinedFunctionsIT.java | 3 +- .../phoenix/end2end/VariableLengthPKIT.java | 38 +- .../phoenix/end2end/index/DropMetadataIT.java | 13 +- .../phoenix/end2end/index/ImmutableIndexIT.java | 20 +- .../end2end/index/IndexExpressionIT.java | 28 +- .../apache/phoenix/end2end/index/IndexIT.java | 58 +- .../phoenix/end2end/index/IndexTestUtil.java | 11 +- .../end2end/index/MutableIndexFailureIT.java | 2 + .../phoenix/end2end/index/MutableIndexIT.java | 21 +- .../phoenix/end2end/salted/SaltedTableIT.java | 2 +- .../phoenix/tx/ParameterizedTransactionIT.java | 518 ++ .../org/apache/phoenix/tx/TransactionIT.java | 589 +- .../org/apache/phoenix/tx/TxCheckpointIT.java | 42 +- .../apache/phoenix/cache/ServerCacheClient.java | 2 + .../org/apache/phoenix/cache/TenantCache.java | 2 +- .../apache/phoenix/cache/TenantCacheImpl.java | 4 +- .../phoenix/compile/CreateTableCompiler.java | 10 +- .../apache/phoenix/compile/DeleteCompiler.java | 2 +- .../phoenix/compile/ExpressionCompiler.java | 18 +- .../apache/phoenix/compile/FromCompiler.java | 54 +- .../apache/phoenix/compile/JoinCompiler.java | 15 +- .../phoenix/compile/ListJarsQueryPlan.java | 6 +- .../apache/phoenix/compile/PostDDLCompiler.java | 11 +- .../compile/PostLocalIndexDDLCompiler.java | 9 +- .../phoenix/compile/ProjectionCompiler.java | 35 +- .../apache/phoenix/compile/TraceQueryPlan.java | 4 +- .../compile/TupleProjectionCompiler.java | 31 +- .../apache/phoenix/compile/UnionCompiler.java | 7 +- .../apache/phoenix/compile/UpsertCompiler.java | 6 +- .../apache/phoenix/compile/WhereCompiler.java | 24 +- .../coprocessor/BaseScannerRegionObserver.java | 59 +- .../GroupedAggregateRegionObserver.java | 42 +- .../coprocessor/HashJoinRegionScanner.java | 25 +- .../coprocessor/MetaDataEndpointImpl.java | 237 +- .../phoenix/coprocessor/MetaDataProtocol.java | 4 +- .../coprocessor/MetaDataRegionObserver.java | 4 +- .../phoenix/coprocessor/ScanRegionObserver.java | 33 +- .../coprocessor/ServerCachingEndpointImpl.java | 2 +- .../coprocessor/ServerCachingProtocol.java | 2 +- .../UngroupedAggregateRegionObserver.java | 65 +- .../coprocessor/generated/PTableProtos.java | 1474 ++++- .../generated/ServerCachingProtos.java | 5125 +++++++++++++++++- .../phoenix/exception/SQLExceptionCode.java | 4 + .../apache/phoenix/execute/BaseQueryPlan.java | 27 +- .../apache/phoenix/execute/CorrelatePlan.java | 2 +- .../apache/phoenix/execute/MutationState.java | 16 +- .../phoenix/execute/SortMergeJoinPlan.java | 2 +- .../apache/phoenix/execute/TupleProjector.java | 72 +- .../apache/phoenix/execute/UnnestArrayPlan.java | 3 +- .../expression/ArrayConstructorExpression.java | 82 +- .../phoenix/expression/ExpressionType.java | 5 +- .../expression/KeyValueColumnExpression.java | 32 +- .../phoenix/expression/LiteralExpression.java | 11 +- .../expression/ProjectedColumnExpression.java | 1 + .../expression/SingleCellColumnExpression.java | 182 + .../SingleCellConstructorExpression.java | 102 + .../function/ArrayElemRefExpression.java | 4 +- .../expression/function/ArrayIndexFunction.java | 4 +- .../expression/util/regex/JONIPattern.java | 18 +- .../visitor/BaseExpressionVisitor.java | 6 + .../visitor/CloneExpressionVisitor.java | 12 + .../CloneNonDeterministicExpressionVisitor.java | 1 + .../expression/visitor/ExpressionVisitor.java | 6 + .../StatelessTraverseAllExpressionVisitor.java | 13 +- .../StatelessTraverseNoExpressionVisitor.java | 13 +- .../phoenix/filter/ColumnProjectionFilter.java | 24 +- ...EncodedQualifiersColumnProjectionFilter.java | 151 + .../MultiEncodedCQKeyValueComparisonFilter.java | 369 ++ .../filter/MultiKeyValueComparisonFilter.java | 6 +- .../SingleCQKeyValueComparisonFilter.java | 3 +- .../filter/SingleKeyValueComparisonFilter.java | 4 +- .../apache/phoenix/hbase/index/ValueGetter.java | 1 + .../hbase/index/util/KeyValueBuilder.java | 1 + .../apache/phoenix/index/IndexMaintainer.java | 530 +- .../phoenix/index/IndexMetaDataCacheClient.java | 1 + .../index/IndexMetaDataCacheFactory.java | 4 +- .../phoenix/index/PhoenixIndexBuilder.java | 2 +- .../apache/phoenix/index/PhoenixIndexCodec.java | 1 + .../index/PhoenixIndexFailurePolicy.java | 2 +- .../phoenix/index/PhoenixIndexMetaData.java | 9 +- .../index/PhoenixTransactionalIndexer.java | 16 +- .../phoenix/iterate/BaseResultIterators.java | 111 +- .../iterate/LookAheadResultIterator.java | 2 +- .../phoenix/iterate/MappedByteBufferQueue.java | 1 + .../phoenix/iterate/OrderedResultIterator.java | 3 +- .../iterate/RegionScannerResultIterator.java | 19 +- .../phoenix/jdbc/PhoenixDatabaseMetaData.java | 14 +- .../apache/phoenix/jdbc/PhoenixResultSet.java | 2 +- .../apache/phoenix/join/HashCacheFactory.java | 2 +- .../mapreduce/FormatToBytesWritableMapper.java | 21 +- .../mapreduce/FormatToKeyValueReducer.java | 30 +- .../index/PhoenixIndexPartialBuildMapper.java | 4 +- .../query/ConnectionQueryServicesImpl.java | 135 +- .../query/ConnectionlessQueryServicesImpl.java | 1 - .../apache/phoenix/query/QueryConstants.java | 59 +- .../org/apache/phoenix/query/QueryServices.java | 4 + .../phoenix/query/QueryServicesOptions.java | 6 +- .../org/apache/phoenix/schema/ColumnRef.java | 16 +- .../phoenix/schema/ColumnValueDecoder.java | 31 + .../phoenix/schema/ColumnValueEncoder.java | 45 + .../apache/phoenix/schema/DelegateColumn.java | 4 + .../apache/phoenix/schema/DelegateTable.java | 24 +- .../apache/phoenix/schema/MetaDataClient.java | 488 +- .../java/org/apache/phoenix/schema/PColumn.java | 4 +- .../apache/phoenix/schema/PColumnFamily.java | 14 +- .../phoenix/schema/PColumnFamilyImpl.java | 49 +- .../org/apache/phoenix/schema/PColumnImpl.java | 29 +- .../apache/phoenix/schema/PMetaDataImpl.java | 4 +- .../java/org/apache/phoenix/schema/PTable.java | 435 +- .../org/apache/phoenix/schema/PTableImpl.java | 480 +- .../org/apache/phoenix/schema/PTableKey.java | 6 +- .../apache/phoenix/schema/ProjectedColumn.java | 12 +- .../org/apache/phoenix/schema/SaltingUtil.java | 2 +- .../apache/phoenix/schema/TableProperty.java | 42 + .../apache/phoenix/schema/tuple/BaseTuple.java | 39 + .../phoenix/schema/tuple/DelegateTuple.java | 7 + .../tuple/EncodedColumnQualiferCellsList.java | 581 ++ .../schema/tuple/MultiKeyValueTuple.java | 1 + .../tuple/PositionBasedMultiKeyValueTuple.java | 90 + .../schema/tuple/PositionBasedResultTuple.java | 125 + .../phoenix/schema/tuple/ResultTuple.java | 20 +- .../org/apache/phoenix/schema/tuple/Tuple.java | 4 + .../phoenix/schema/types/PArrayDataType.java | 340 +- .../schema/types/PArrayDataTypeDecoder.java | 102 + .../schema/types/PArrayDataTypeEncoder.java | 170 + .../apache/phoenix/util/EncodedColumnsUtil.java | 205 + .../java/org/apache/phoenix/util/IndexUtil.java | 76 +- .../org/apache/phoenix/util/KeyValueUtil.java | 2 - .../org/apache/phoenix/util/MetaDataUtil.java | 5 + .../org/apache/phoenix/util/PhoenixRuntime.java | 12 +- .../org/apache/phoenix/util/ResultUtil.java | 60 - .../java/org/apache/phoenix/util/ScanUtil.java | 29 +- .../org/apache/phoenix/util/SchemaUtil.java | 29 +- .../apache/phoenix/cache/TenantCacheTest.java | 6 +- .../phoenix/compile/HavingCompilerTest.java | 2 +- .../phoenix/compile/QueryCompilerTest.java | 28 +- .../phoenix/compile/QueryOptimizerTest.java | 51 + .../compile/SelectStatementRewriterTest.java | 11 +- .../phoenix/compile/WhereCompilerTest.java | 44 +- .../phoenix/execute/CorrelatePlanTest.java | 12 +- .../execute/LiteralResultIteratorPlanTest.java | 12 +- .../phoenix/execute/MutationStateTest.java | 4 +- .../phoenix/execute/UnnestArrayPlanTest.java | 8 +- .../ArrayConstructorExpressionTest.java | 20 +- .../expression/ColumnExpressionTest.java | 27 +- .../phoenix/index/IndexMaintainerTest.java | 7 +- .../iterate/AggregateResultScannerTest.java | 2 +- .../query/BaseConnectionlessQueryTest.java | 18 +- .../java/org/apache/phoenix/query/BaseTest.java | 61 +- .../phoenix/query/ConnectionlessTest.java | 18 +- .../EncodedColumnQualifierCellsListTest.java | 608 +++ .../schema/ImmutableStorageSchemeTest.java | 182 + .../schema/types/PDataTypeForArraysTest.java | 38 +- .../apache/phoenix/util/PhoenixRuntimeTest.java | 7 +- .../util/QualifierEncodingSchemeTest.java | 119 + .../java/org/apache/phoenix/util/TestUtil.java | 89 +- phoenix-protocol/src/main/PTable.proto | 9 + .../src/main/ServerCachingService.proto | 35 + pom.xml | 2 +- 202 files changed, 15812 insertions(+), 3071 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6a82e29/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateQueryIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateQueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateQueryIT.java index c689373..a205814 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateQueryIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateQueryIT.java @@ -17,12 +17,10 @@ */ package org.apache.phoenix.end2end; -import static org.apache.phoenix.util.TestUtil.ATABLE_NAME; import static org.apache.phoenix.util.TestUtil.A_VALUE; import static org.apache.phoenix.util.TestUtil.B_VALUE; import static org.apache.phoenix.util.TestUtil.C_VALUE; import static org.apache.phoenix.util.TestUtil.E_VALUE; -import static org.apache.phoenix.util.TestUtil.ROW3; import static org.apache.phoenix.util.TestUtil.ROW4; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; @@ -47,55 +45,13 @@ import org.junit.Test; public class AggregateQueryIT extends BaseQueryIT { - public AggregateQueryIT(String indexDDL) { - super(indexDDL); - } - - @Test - public void testSumOverNullIntegerColumn() throws Exception { - String query = "SELECT sum(a_integer) FROM aTable a"; - Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); - props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 20)); - Connection conn = DriverManager.getConnection(getUrl(), props); - conn.setAutoCommit(true); - conn.createStatement().execute("UPSERT INTO atable(organization_id,entity_id,a_integer) VALUES('" + getOrganizationId() + "','" + ROW3 + "',NULL)"); - props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 30)); - Connection conn1 = DriverManager.getConnection(getUrl(), props); - analyzeTable(conn1, "ATABLE"); - conn1.close(); - props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 50)); - conn = DriverManager.getConnection(getUrl(), props); - try { - PreparedStatement statement = conn.prepareStatement(query); - ResultSet rs = statement.executeQuery(); - assertTrue (rs.next()); - assertEquals(42, rs.getInt(1)); - assertFalse(rs.next()); - } finally { - conn.close(); - } - props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 70)); - conn = DriverManager.getConnection(getUrl(), props); - conn.setAutoCommit(true); - conn.createStatement().execute("UPSERT INTO atable(organization_id,entity_id,a_integer) SELECT organization_id, entity_id, CAST(null AS integer) FROM atable"); - - props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 90)); - conn = DriverManager.getConnection(getUrl(), props); - try { - PreparedStatement statement = conn.prepareStatement(query); - ResultSet rs = statement.executeQuery(); - assertTrue (rs.next()); - assertEquals(0, rs.getInt(1)); - assertTrue(rs.wasNull()); - assertFalse(rs.next()); - } finally { - conn.close(); - } + public AggregateQueryIT(String indexDDL, boolean mutable, boolean columnEncoded) { + super(indexDDL, mutable, columnEncoded); } @Test public void testGroupByPlusOne() throws Exception { - String query = "SELECT a_integer+1 FROM aTable WHERE organization_id=? and a_integer = 5 GROUP BY a_integer+1"; + String query = "SELECT a_integer+1 FROM " + tableName + " WHERE organization_id=? and a_integer = 5 GROUP BY a_integer+1"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 Connection conn = DriverManager.getConnection(getUrl(), props); @@ -114,7 +70,7 @@ public class AggregateQueryIT extends BaseQueryIT { @Test public void testSplitWithCachedMeta() throws Exception { // Tests that you don't get an ambiguous column exception when using the same alias as the column name - String query = "SELECT a_string, b_string, count(1) FROM atable WHERE organization_id=? and entity_id<=? GROUP BY a_string,b_string"; + String query = "SELECT a_string, b_string, count(1) FROM " + tableName + " WHERE organization_id=? and entity_id<=? GROUP BY a_string,b_string"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 Connection conn = DriverManager.getConnection(getUrl(), props); @@ -138,12 +94,12 @@ public class AggregateQueryIT extends BaseQueryIT { assertEquals(1, rs.getLong(3)); assertFalse(rs.next()); - byte[] tableName = Bytes.toBytes(ATABLE_NAME); + byte[] tableNameBytes = Bytes.toBytes(tableName); admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin(); - HTable htable = (HTable) conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(tableName); + HTable htable = (HTable) conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(tableNameBytes); htable.clearRegionCache(); int nRegions = htable.getRegionLocations().size(); - admin.split(tableName, ByteUtil.concat(Bytes.toBytes(tenantId), Bytes.toBytes("00A" + Character.valueOf((char) ('3' + nextRunCount())) + ts))); // vary split point with test run + admin.split(tableNameBytes, ByteUtil.concat(Bytes.toBytes(tenantId), Bytes.toBytes("00A" + Character.valueOf((char) ('3' + nextRunCount())) + ts))); // vary split point with test run int retryCount = 0; do { Thread.sleep(2000); @@ -177,7 +133,7 @@ public class AggregateQueryIT extends BaseQueryIT { @Test public void testCountIsNull() throws Exception { - String query = "SELECT count(1) FROM aTable WHERE X_DECIMAL is null"; + String query = "SELECT count(1) FROM " + tableName + " WHERE X_DECIMAL is null"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 Connection conn = DriverManager.getConnection(getUrl(), props); @@ -195,7 +151,7 @@ public class AggregateQueryIT extends BaseQueryIT { @Test public void testCountWithNoScanRanges() throws Exception { - String query = "SELECT count(1) FROM aTable WHERE organization_id = 'not_existing_organization_id'"; + String query = "SELECT count(1) FROM " + tableName + " WHERE organization_id = 'not_existing_organization_id'"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 Connection conn = DriverManager.getConnection(getUrl(), props); @@ -205,19 +161,19 @@ public class AggregateQueryIT extends BaseQueryIT { assertTrue(rs.next()); assertEquals(0, rs.getLong(1)); assertFalse(rs.next()); - query = "SELECT count(1) FROM aTable WHERE organization_id = 'not_existing_organization_id' having count(*)>0"; + query = "SELECT count(1) FROM " + tableName + " WHERE organization_id = 'not_existing_organization_id' having count(*)>0"; rs = conn.prepareStatement(query).executeQuery(); assertFalse(rs.next()); - query = "SELECT count(1) FROM aTable WHERE organization_id = 'not_existing_organization_id' limit 1 offset 1"; + query = "SELECT count(1) FROM " + tableName + " WHERE organization_id = 'not_existing_organization_id' limit 1 offset 1"; rs = conn.prepareStatement(query).executeQuery(); assertFalse(rs.next()); - query = "SELECT count(1),123 FROM aTable WHERE organization_id = 'not_existing_organization_id'"; + query = "SELECT count(1),123 FROM " + tableName + " WHERE organization_id = 'not_existing_organization_id'"; rs = conn.prepareStatement(query).executeQuery(); assertTrue(rs.next()); assertEquals(0, rs.getLong(1)); assertEquals("123", rs.getString(2)); assertFalse(rs.next()); - query = "SELECT count(1),sum(x_decimal) FROM aTable WHERE organization_id = 'not_existing_organization_id'"; + query = "SELECT count(1),sum(x_decimal) FROM " + tableName + " WHERE organization_id = 'not_existing_organization_id'"; rs = conn.prepareStatement(query).executeQuery(); assertTrue(rs.next()); assertEquals(0, rs.getLong(1)); @@ -230,7 +186,7 @@ public class AggregateQueryIT extends BaseQueryIT { @Test public void testCountIsNotNull() throws Exception { - String query = "SELECT count(1) FROM aTable WHERE X_DECIMAL is not null"; + String query = "SELECT count(1) FROM " + tableName + " WHERE X_DECIMAL is not null"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 Connection conn = DriverManager.getConnection(getUrl(), props); @@ -250,7 +206,7 @@ public class AggregateQueryIT extends BaseQueryIT { */ @Test public void testInFilterOnKey() throws Exception { - String query = "SELECT count(entity_id) FROM ATABLE WHERE organization_id IN (?,?)"; + String query = "SELECT count(entity_id) FROM " + tableName + " WHERE organization_id IN (?,?)"; String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + (ts + 5); // Run query at timestamp 5 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); Connection conn = DriverManager.getConnection(url, props); http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6a82e29/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterMultiTenantTableWithViewsIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterMultiTenantTableWithViewsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterMultiTenantTableWithViewsIT.java index d6f3a7f..7b4ff68 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterMultiTenantTableWithViewsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterMultiTenantTableWithViewsIT.java @@ -476,14 +476,15 @@ public class AlterMultiTenantTableWithViewsIT extends ParallelStatsDisabledIT { // For a diverged view, only base table's pk column will be added and that too at the end. assertTableDefinition(conn, divergedView, PTableType.VIEW, baseTable, 2, 7, DIVERGED_VIEW_BASE_COLUMN_COUNT, "TENANT_ID", "PK1", "V1", "V3", "VIEW_COL1", "VIEW_COL2", "PK2"); - // Add existing column VIEW_COL2 to the base table - alterBaseTable = "ALTER TABLE " + baseTable + " ADD VIEW_COL2 CHAR(256)"; - conn.createStatement().execute(alterBaseTable); - - // For the non-diverged view, adding the column VIEW_COL2 will end up changing its ordinal position in the view. - assertTableDefinition(conn, view1, PTableType.VIEW, baseTable, 2, 9, 8, "TENANT_ID", "PK1", "V1", "V2", "V3", "KV", "PK2", "VIEW_COL2", "VIEW_COL1"); - // For the diverged view, adding the column VIEW_COL2 will not change its ordinal position in the view. It also won't change the base column count or the sequence number - assertTableDefinition(conn, divergedView, PTableType.VIEW, baseTable, 2, 7, DIVERGED_VIEW_BASE_COLUMN_COUNT, "TENANT_ID", "PK1", "V1", "V3", "VIEW_COL1", "VIEW_COL2", "PK2"); + // Adding existing column VIEW_COL2 to the base table isn't allowed. + try { + alterBaseTable = "ALTER TABLE " + baseTable + " ADD VIEW_COL2 CHAR(256)"; + conn.createStatement().execute(alterBaseTable); + fail(); + } + catch (SQLException e) { + assertEquals("Unexpected exception", SQLExceptionCode.CANNOT_MUTATE_TABLE.getErrorCode(), e.getErrorCode()); + } } } @@ -500,13 +501,13 @@ public class AlterMultiTenantTableWithViewsIT extends ParallelStatsDisabledIT { tenant1Conn.createStatement().execute(view1DDL); } - assertTableDefinition(conn, baseTable, PTableType.TABLE, null, 0, 5, BASE_TABLE_BASE_COLUMN_COUNT, "TENANT_ID", "PK1", "V1", "V2", "V3"); + assertTableDefinition(conn, baseTable, PTableType.TABLE, null, 1, 5, BASE_TABLE_BASE_COLUMN_COUNT, "TENANT_ID", "PK1", "V1", "V2", "V3"); assertTableDefinition(conn, view1, PTableType.VIEW, baseTable, 0, 7, 5, "TENANT_ID", "PK1", "V1", "V2", "V3", "VIEW_COL1", "VIEW_COL2"); String alterBaseTable = "ALTER TABLE " + baseTable + " ADD KV VARCHAR, PK2 VARCHAR PRIMARY KEY"; conn.createStatement().execute(alterBaseTable); - assertTableDefinition(conn, baseTable, PTableType.TABLE, null, 1, 7, BASE_TABLE_BASE_COLUMN_COUNT, "TENANT_ID", "PK1", "V1", "V2", "V3", "KV", "PK2"); + assertTableDefinition(conn, baseTable, PTableType.TABLE, null, 2, 7, BASE_TABLE_BASE_COLUMN_COUNT, "TENANT_ID", "PK1", "V1", "V2", "V3", "KV", "PK2"); assertTableDefinition(conn, view1, PTableType.VIEW, baseTable, 1, 9, 7, "TENANT_ID", "PK1", "V1", "V2", "V3", "KV", "PK2", "VIEW_COL1", "VIEW_COL2"); // verify that the both columns were added to view1 @@ -530,13 +531,13 @@ public class AlterMultiTenantTableWithViewsIT extends ParallelStatsDisabledIT { tenant1Conn.createStatement().execute(view1DDL); } - assertTableDefinition(conn, baseTable, PTableType.TABLE, null, 0, 5, BASE_TABLE_BASE_COLUMN_COUNT, "TENANT_ID", "PK1", "V1", "V2", "V3"); + assertTableDefinition(conn, baseTable, PTableType.TABLE, null, 1, 5, BASE_TABLE_BASE_COLUMN_COUNT, "TENANT_ID", "PK1", "V1", "V2", "V3"); assertTableDefinition(conn, view1, PTableType.VIEW, baseTable, 0, 7, 5, "TENANT_ID", "PK1", "V1", "V2", "V3", "VIEW_COL1", "VIEW_COL2"); String alterBaseTable = "ALTER TABLE " + baseTable + " DROP COLUMN V2"; conn.createStatement().execute(alterBaseTable); - assertTableDefinition(conn, baseTable, PTableType.TABLE, null, 1, 4, BASE_TABLE_BASE_COLUMN_COUNT, "TENANT_ID", "PK1", "V1", "V3"); + assertTableDefinition(conn, baseTable, PTableType.TABLE, null, 2, 4, BASE_TABLE_BASE_COLUMN_COUNT, "TENANT_ID", "PK1", "V1", "V3"); assertTableDefinition(conn, view1, PTableType.VIEW, baseTable, 1, 6, 4, "TENANT_ID", "PK1", "V1", "V3", "VIEW_COL1", "VIEW_COL2"); // verify that the dropped columns aren't visible http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6a82e29/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java index fee7316..d13b8d2 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java @@ -18,6 +18,15 @@ package org.apache.phoenix.end2end; import static org.apache.hadoop.hbase.HColumnDescriptor.DEFAULT_REPLICATION_SCOPE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_FAMILY; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER_COUNTER; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM; +import static org.apache.phoenix.query.QueryConstants.DEFAULT_COLUMN_FAMILY; +import static org.apache.phoenix.query.QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.apache.phoenix.util.TestUtil.closeConnection; import static org.apache.phoenix.util.TestUtil.closeStatement; @@ -35,6 +44,8 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.Arrays; +import java.util.Collection; import java.util.Properties; import org.apache.hadoop.hbase.HColumnDescriptor; @@ -53,6 +64,8 @@ import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTable.EncodedCQCounter; +import org.apache.phoenix.schema.PTable.QualifierEncodingScheme; import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.schema.TableNotFoundException; import org.apache.phoenix.util.IndexUtil; @@ -60,6 +73,9 @@ import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.SchemaUtil; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; /** * @@ -72,14 +88,28 @@ import org.junit.Test; * or at the end of test class. * */ +@RunWith(Parameterized.class) public class AlterTableIT extends ParallelStatsDisabledIT { private String schemaName; private String dataTableName; private String indexTableName; private String localIndexTableName; + private String viewName; private String dataTableFullName; private String indexTableFullName; private String localIndexTableFullName; + private String tableDDLOptions; + private final boolean columnEncoded; + + public AlterTableIT(boolean columnEncoded) { + this.columnEncoded = columnEncoded; + this.tableDDLOptions = columnEncoded ? "" : "COLUMN_ENCODED_BYTES=0"; + } + + @Parameters(name="AlterTableIT_columnEncoded={0}") // name is used by failsafe as file name in reports + public static Collection<Boolean> data() { + return Arrays.asList( false, true); + } @Before public void setupTableNames() throws Exception { @@ -90,6 +120,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName); indexTableFullName = SchemaUtil.getTableName(schemaName, indexTableName); localIndexTableFullName = SchemaUtil.getTableName(schemaName, localIndexTableName); + viewName = generateUniqueName(); } @Test @@ -101,7 +132,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { try { String ddl = "CREATE TABLE " + dataTableFullName + " (a_string varchar not null, a_binary varbinary not null, col1 integer" + - " CONSTRAINT pk PRIMARY KEY (a_string, a_binary))\n"; + " CONSTRAINT pk PRIMARY KEY (a_string, a_binary)) " + tableDDLOptions; createTestTable(getUrl(), ddl); ddl = "ALTER TABLE " + dataTableFullName + " ADD b_string VARCHAR NULL PRIMARY KEY"; @@ -142,7 +173,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { try { String ddl = "CREATE TABLE " + dataTableFullName + " (a_string varchar not null, col1 integer" + - " CONSTRAINT pk PRIMARY KEY (a_string))\n"; + " CONSTRAINT pk PRIMARY KEY (a_string)) " + tableDDLOptions; conn.createStatement().execute(ddl); String dml = "UPSERT INTO " + dataTableFullName + " VALUES(?)"; @@ -209,7 +240,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { Connection conn = DriverManager.getConnection(getUrl(), props); String ddl = "CREATE TABLE " + dataTableFullName + " (a_string varchar not null, col1 integer" + - " CONSTRAINT pk PRIMARY KEY (a_string))\n"; + " CONSTRAINT pk PRIMARY KEY (a_string)) " + tableDDLOptions; try { conn.createStatement().execute(ddl); conn.createStatement().execute("ALTER TABLE " + dataTableFullName + " ADD CF.col2 integer CF.IN_MEMORY=true"); @@ -231,8 +262,31 @@ public class AlterTableIT extends ParallelStatsDisabledIT { assertEquals(exists, rs.next()); } + @Test - public void testDropIndexedColumn() throws Exception { + public void testDropIndexedColumnImmutableIndex() throws Exception { + helpTestDropIndexedColumn(true); + } + + @Test + public void testDropIndexedColumnMutableIndex() throws Exception { + helpTestDropIndexedColumn(false); + } + + private String generateDDLOptions(String options) { + StringBuilder sb = new StringBuilder(); + if (!options.isEmpty()) { + sb.append(options); + } + if (!tableDDLOptions.isEmpty()) { + if (sb.length()!=0) + sb.append(","); + sb.append(tableDDLOptions); + } + return sb.toString(); + } + + private void helpTestDropIndexedColumn(boolean immutable) throws Exception { String query; ResultSet rs; PreparedStatement stmt; @@ -244,7 +298,9 @@ public class AlterTableIT extends ParallelStatsDisabledIT { // make sure that the tables are empty, but reachable conn.createStatement().execute( "CREATE TABLE " + dataTableFullName - + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)"); + + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) " + + generateDDLOptions(immutable ? "IMMUTABLE_ROWS = true" : "") + + (!columnEncoded ? ",IMMUTABLE_STORAGE_SCHEME=" + PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN : "")); query = "SELECT * FROM " + dataTableFullName; rs = conn.createStatement().executeQuery(query); assertFalse(rs.next()); @@ -304,7 +360,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { // make sure that the tables are empty, but reachable conn.createStatement().execute( "CREATE TABLE " + dataTableFullName - + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR, v3 VARCHAR)"); + + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR, v3 VARCHAR) " + tableDDLOptions); String dataTableQuery = "SELECT * FROM " + dataTableFullName; rs = conn.createStatement().executeQuery(dataTableQuery); assertFalse(rs.next()); @@ -419,7 +475,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { // make sure that the tables are empty, but reachable conn.createStatement().execute( "CREATE TABLE " + dataTableFullName - + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)"); + + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) " + tableDDLOptions); query = "SELECT * FROM " + dataTableFullName; rs = conn.createStatement().executeQuery(query); assertFalse(rs.next()); @@ -515,7 +571,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { " B.M_TEXT VARCHAR\n" + " CONSTRAINT ROWKEY PRIMARY KEY\n" + "(SENDER_ID,RECIPIENT_ID,M_TIMESTAMP DESC,ROW_ID))\n" + - "SALT_BUCKETS=4"; + generateDDLOptions("SALT_BUCKETS=4"); conn.createStatement().execute(ddl); ddl = "ALTER TABLE " + dataTableFullName + " SET IMMUTABLE_ROWS=true"; @@ -549,7 +605,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { " B.M_TEXT VARCHAR\n" + " CONSTRAINT ROWKEY PRIMARY KEY\n" + "(SENDER_ID,RECIPIENT_ID,M_TIMESTAMP DESC,ROW_ID))\n" + - "SALT_BUCKETS=4"; + generateDDLOptions("SALT_BUCKETS=4"); conn.createStatement().execute(ddl); ddl = "ALTER TABLE " + dataTableFullName + " DROP COLUMN B.JSON"; @@ -572,7 +628,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { try { String ddl = "CREATE TABLE " + dataTableFullName + " (a_string varchar not null, col1 integer" + - " CONSTRAINT pk PRIMARY KEY (a_string))\n"; + " CONSTRAINT pk PRIMARY KEY (a_string)) " + tableDDLOptions; conn.createStatement().execute(ddl); String dml = "UPSERT INTO " + dataTableFullName + " VALUES(?)"; @@ -679,7 +735,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { conn.setAutoCommit(false); try { String ddl = "CREATE TABLE " + dataTableFullName + " " + " (a_string varchar not null, col1 integer, cf1.col2 integer" - + " CONSTRAINT pk PRIMARY KEY (a_string))\n"; + + " CONSTRAINT pk PRIMARY KEY (a_string)) " + tableDDLOptions; conn.createStatement().execute(ddl); ddl = "ALTER TABLE " + dataTableFullName + " DROP COLUMN col1"; @@ -702,7 +758,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { conn.setAutoCommit(false); try { String ddl = "CREATE TABLE " + dataTableFullName + " " + " (a_string varchar not null, col1 integer, cf1.col2 integer" - + " CONSTRAINT pk PRIMARY KEY (a_string))\n"; + + " CONSTRAINT pk PRIMARY KEY (a_string)) " + tableDDLOptions; stmt = conn.prepareStatement(ddl); stmt.execute(); } finally { @@ -737,7 +793,9 @@ public class AlterTableIT extends ParallelStatsDisabledIT { .execute( "CREATE TABLE " + dataTableFullName + " (a_string varchar not null, col1 integer, cf1.col2 integer, col3 integer , cf2.col4 integer " - + " CONSTRAINT pk PRIMARY KEY (a_string)) immutable_rows=true, disable_wal=true "); + + " CONSTRAINT pk PRIMARY KEY (a_string)) " + + generateDDLOptions("immutable_rows=true, disable_wal=true" + + (!columnEncoded ? ",IMMUTABLE_STORAGE_SCHEME=" + PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN : ""))); Connection conn2 = DriverManager.getConnection(getUrl(), props); String query = "SELECT * FROM " + dataTableFullName; @@ -769,7 +827,9 @@ public class AlterTableIT extends ParallelStatsDisabledIT { .execute( "CREATE TABLE " + dataTableFullName + " (a_string varchar not null, col1 integer, cf1.col2 integer, col3 integer , cf2.col4 integer " - + " CONSTRAINT pk PRIMARY KEY (a_string)) immutable_rows=true"); + + " CONSTRAINT pk PRIMARY KEY (a_string))" + + generateDDLOptions("immutable_rows=true" + + (!columnEncoded ? ",IMMUTABLE_STORAGE_SCHEME=" + PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN : ""))); Connection conn2 = DriverManager.getConnection(getUrl(), props); String query = "SELECT * FROM " + dataTableFullName; @@ -802,7 +862,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { .execute( "CREATE TABLE " + dataTableFullName + " (a_string varchar not null, col1 integer, cf1.col2 integer, col3 integer , cf2.col4 integer " - + " CONSTRAINT pk PRIMARY KEY (a_string))"); + + " CONSTRAINT pk PRIMARY KEY (a_string)) " + tableDDLOptions); Connection conn2 = DriverManager.getConnection(getUrl(), props); String query = "SELECT * FROM " + dataTableFullName; @@ -840,7 +900,9 @@ public class AlterTableIT extends ParallelStatsDisabledIT { .execute( "CREATE TABLE " + dataTableFullName + " (a_string varchar not null, col1 integer, cf1.col2 integer, col3 integer , cf2.col4 integer " - + " CONSTRAINT pk PRIMARY KEY (a_string)) immutable_rows=true , SALT_BUCKETS=3 "); + + " CONSTRAINT pk PRIMARY KEY (a_string)) " + + generateDDLOptions("immutable_rows=true , SALT_BUCKETS=3 " + + (!columnEncoded ? ",IMMUTABLE_STORAGE_SCHEME=" + PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN : ""))); String query = "SELECT * FROM " + dataTableFullName; ResultSet rs = conn.createStatement().executeQuery(query); @@ -929,7 +991,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { // here we insert into the orig schema with one column Connection conn1 = DriverManager.getConnection(getUrl(), props); - conn1.createStatement().execute("create table " + dataTableFullName + "(id VARCHAR PRIMARY KEY, field1 BIGINT)"); + conn1.createStatement().execute("create table " + dataTableFullName + "(id VARCHAR PRIMARY KEY, field1 BIGINT) " + tableDDLOptions); PreparedStatement stmtInsert1 = conn1.prepareStatement("upsert into " + dataTableFullName + " (id, field1) values ( ?, ?)"); stmtInsert1.setString(1, "key1"); stmtInsert1.setLong(2, 1L); @@ -961,7 +1023,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { +"CREATED_DATE DATE,\n" +"CREATION_TIME BIGINT,\n" +"LAST_USED DATE,\n" - +"CONSTRAINT PK PRIMARY KEY (ID1, ID2))"; + +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) " + tableDDLOptions; Connection conn1 = DriverManager.getConnection(getUrl(), props); conn1.createStatement().execute(ddl); ddl = "ALTER TABLE " + dataTableFullName + " ADD STRING VARCHAR, STRING_DATA_TYPES VARCHAR"; @@ -978,7 +1040,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); String ddl = "CREATE TABLE " + dataTableFullName + " (\n" +"ID VARCHAR(15) PRIMARY KEY,\n" - +"COL1 BIGINT)"; + +"COL1 BIGINT) " + tableDDLOptions; Connection conn1 = DriverManager.getConnection(getUrl(), props); conn1.createStatement().execute(ddl); conn1.createStatement().execute("CREATE INDEX " + indexTableName + " ON " + dataTableFullName + "(COL1)"); @@ -1035,7 +1097,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { + "COL1 BIGINT," + "COL2 BIGINT," + "COL3 BIGINT," - + "COL4 BIGINT)"; + + "COL4 BIGINT) " + tableDDLOptions; Connection conn1 = DriverManager.getConnection(getUrl(), props); conn1.createStatement().execute(ddl); conn1.createStatement().execute("CREATE INDEX " + indexTableName + " ON " + dataTableFullName + "(COL1) INCLUDE (COL2,COL3,COL4)"); @@ -1091,7 +1153,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { +"CREATED_DATE DATE,\n" +"CREATION_TIME BIGINT,\n" +"LAST_USED DATE,\n" - +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) SALT_BUCKETS = 8"; + +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) " + generateDDLOptions("SALT_BUCKETS = 8"); Connection conn1 = DriverManager.getConnection(getUrl(), props); conn1.createStatement().execute(ddl); ddl = "ALTER TABLE " + dataTableFullName + " SET REPLICATION_SCOPE=1"; @@ -1114,7 +1176,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { +"CREATED_DATE DATE,\n" +"CREATION_TIME BIGINT,\n" +"LAST_USED DATE,\n" - +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) SALT_BUCKETS = 8"; + +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) " + generateDDLOptions("SALT_BUCKETS = 8"); Connection conn1 = DriverManager.getConnection(getUrl(), props); conn1.createStatement().execute(ddl); ddl = "ALTER TABLE " + dataTableFullName + " SET COMPACTION_ENABLED=FALSE"; @@ -1136,7 +1198,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { +"CREATED_DATE DATE,\n" +"CREATION_TIME BIGINT,\n" +"LAST_USED DATE,\n" - +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) SALT_BUCKETS = 8"; + +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) " + generateDDLOptions("SALT_BUCKETS = 8"); Connection conn1 = DriverManager.getConnection(getUrl(), props); conn1.createStatement().execute(ddl); ddl = "ALTER TABLE " + dataTableFullName + " SET COMPACTION_ENABLED = FALSE, REPLICATION_SCOPE = 1"; @@ -1160,7 +1222,8 @@ public class AlterTableIT extends ParallelStatsDisabledIT { +"CREATED_DATE DATE,\n" +"CF1.CREATION_TIME BIGINT,\n" +"CF2.LAST_USED DATE,\n" - +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) IMMUTABLE_ROWS=true"; + +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) " + generateDDLOptions("IMMUTABLE_ROWS=true" + + (!columnEncoded ? ",IMMUTABLE_STORAGE_SCHEME=" + PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN : "")); Connection conn = DriverManager.getConnection(getUrl(), props); conn.createStatement().execute(ddl); assertImmutableRows(conn, dataTableFullName, true); @@ -1202,7 +1265,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { +"CREATED_DATE DATE,\n" +"CREATION_TIME BIGINT,\n" +"LAST_USED DATE,\n" - +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) SALT_BUCKETS = 8"; + +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) " + generateDDLOptions("SALT_BUCKETS = 8"); Connection conn1 = DriverManager.getConnection(getUrl(), props); conn1.createStatement().execute(ddl); ddl = "ALTER TABLE " + dataTableFullName + " SET CF.COMPACTION_ENABLED = FALSE"; @@ -1223,7 +1286,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { +"CREATED_DATE DATE,\n" +"CREATION_TIME BIGINT,\n" +"LAST_USED DATE,\n" - +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) SALT_BUCKETS = 8"; + +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) " + generateDDLOptions("SALT_BUCKETS = 8"); Connection conn1 = DriverManager.getConnection(getUrl(), props); conn1.createStatement().execute(ddl); ddl = "ALTER TABLE " + dataTableFullName + " SET CF.DISABLE_WAL = TRUE"; @@ -1244,7 +1307,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { +"CREATED_DATE DATE,\n" +"CREATION_TIME BIGINT,\n" +"CF.LAST_USED DATE,\n" - +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) SALT_BUCKETS = 8"; + +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) " + generateDDLOptions("SALT_BUCKETS = 8"); Connection conn1 = DriverManager.getConnection(getUrl(), props); conn1.createStatement().execute(ddl); ddl = "ALTER TABLE " + dataTableFullName + " SET CF.TTL = 86400"; @@ -1265,7 +1328,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { +"CREATED_DATE DATE,\n" +"CREATION_TIME BIGINT,\n" +"LAST_USED DATE,\n" - +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) SALT_BUCKETS = 8"; + +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) " + generateDDLOptions("SALT_BUCKETS = 8"); Connection conn1 = DriverManager.getConnection(getUrl(), props); conn1.createStatement().execute(ddl); ddl = "ALTER TABLE " + dataTableFullName + " SET CF.REPLICATION_SCOPE = 1"; @@ -1286,7 +1349,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { +"CREATED_DATE DATE,\n" +"CREATION_TIME BIGINT,\n" +"LAST_USED DATE,\n" - +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) SALT_BUCKETS = 8"; + +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) " + generateDDLOptions(" SALT_BUCKETS = 8"); Connection conn1 = DriverManager.getConnection(getUrl(), props); conn1.createStatement().execute(ddl); ddl = "ALTER TABLE " + dataTableFullName + " SET DEFAULT_COLUMN_FAMILY = 'A'"; @@ -1307,19 +1370,19 @@ public class AlterTableIT extends ParallelStatsDisabledIT { +"CREATED_DATE DATE,\n" +"CREATION_TIME BIGINT,\n" +"LAST_USED DATE,\n" - +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) SALT_BUCKETS = 8"; + +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) " + generateDDLOptions("SALT_BUCKETS = 8"); Connection conn1 = DriverManager.getConnection(getUrl(), props); conn1.createStatement().execute(ddl); - ddl = "CREATE VIEW v AS SELECT * FROM " + dataTableFullName + " WHERE CREATION_TIME = 1"; + ddl = "CREATE VIEW " + viewName + " AS SELECT * FROM " + dataTableFullName + " WHERE CREATION_TIME = 1"; conn1.createStatement().execute(ddl); - ddl = "ALTER VIEW v SET REPLICATION_SCOPE = 1"; + ddl = "ALTER VIEW " + viewName + " SET REPLICATION_SCOPE = 1"; try { conn1.createStatement().execute(ddl); fail(); } catch (SQLException e) { assertEquals(SQLExceptionCode.VIEW_WITH_PROPERTIES.getErrorCode(), e.getErrorCode()); } - ddl = "ALTER VIEW v SET COMPACTION_ENABLED = FALSE"; + ddl = "ALTER VIEW " + viewName + " SET COMPACTION_ENABLED = FALSE"; try { conn1.createStatement().execute(ddl); fail(); @@ -1337,7 +1400,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { +"CREATED_DATE DATE,\n" +"CREATION_TIME BIGINT,\n" +"LAST_USED DATE,\n" - +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) SALT_BUCKETS = 8"; + +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) " + generateDDLOptions("SALT_BUCKETS = 8"); Connection conn1 = DriverManager.getConnection(getUrl(), props); conn1.createStatement().execute(ddl); String viewFullName = SchemaUtil.getTableName(schemaName, generateUniqueName()); @@ -1372,7 +1435,8 @@ public class AlterTableIT extends ParallelStatsDisabledIT { +"CREATED_DATE DATE,\n" +"CREATION_TIME BIGINT,\n" +"CF.LAST_USED DATE,\n" - +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) IMMUTABLE_ROWS=true, DEFAULT_COLUMN_FAMILY = 'XYZ'"; + +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) " + generateDDLOptions("IMMUTABLE_ROWS=true, DEFAULT_COLUMN_FAMILY = 'XYZ'" + + (!columnEncoded ? ",IMMUTABLE_STORAGE_SCHEME=" + PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN : "")); Connection conn = DriverManager.getConnection(getUrl(), props); conn.createStatement().execute(ddl); assertImmutableRows(conn, dataTableFullName, true); @@ -1406,7 +1470,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { String ddl = "CREATE TABLE " + dataTableFullName + " (a_string varchar not null, col1 integer, CF.col2 integer" + - " CONSTRAINT pk PRIMARY KEY (a_string))\n"; + " CONSTRAINT pk PRIMARY KEY (a_string)) " + tableDDLOptions; try { conn.createStatement().execute(ddl); conn.createStatement().execute( @@ -1432,7 +1496,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { String ddl = "CREATE TABLE " + dataTableFullName + " " + " (a_string varchar not null, col1 integer, CF1.col2 integer" + - " CONSTRAINT pk PRIMARY KEY (a_string))\n"; + " CONSTRAINT pk PRIMARY KEY (a_string)) " + tableDDLOptions; try { conn.createStatement().execute(ddl); conn.createStatement() @@ -1466,7 +1530,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { String ddl = "CREATE TABLE " + dataTableFullName + " " + " (a_string varchar not null, col1 integer, CF1.col2 integer" + - " CONSTRAINT pk PRIMARY KEY (a_string)) DEFAULT_COLUMN_FAMILY = 'XYZ'\n"; + " CONSTRAINT pk PRIMARY KEY (a_string)) " + generateDDLOptions("DEFAULT_COLUMN_FAMILY = 'XYZ'"); try { conn.createStatement().execute(ddl); conn.createStatement() @@ -1500,7 +1564,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { String ddl = "CREATE TABLE " + dataTableFullName + " " + " (a_string varchar not null, col1 integer, CF1.col2 integer" + - " CONSTRAINT pk PRIMARY KEY (a_string)) DEFAULT_COLUMN_FAMILY = 'XYZ'\n"; + " CONSTRAINT pk PRIMARY KEY (a_string)) "+ generateDDLOptions("DEFAULT_COLUMN_FAMILY = 'XYZ'"); try { conn.createStatement().execute(ddl); try { @@ -1523,7 +1587,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { String ddl = "CREATE TABLE " + dataTableFullName + " (a_string varchar not null, col1 integer, CF1.col2 integer, CF2.col3 integer" + - " CONSTRAINT pk PRIMARY KEY (a_string)) DEFAULT_COLUMN_FAMILY = 'XYZ'\n"; + " CONSTRAINT pk PRIMARY KEY (a_string)) " + generateDDLOptions("DEFAULT_COLUMN_FAMILY = 'XYZ' "); try { conn.createStatement().execute(ddl); conn.createStatement() @@ -1560,7 +1624,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { String ddl = "CREATE TABLE " + dataTableFullName + " (a_string varchar not null, col1 integer, CF1.col2 integer" + - " CONSTRAINT pk PRIMARY KEY (a_string)) DEFAULT_COLUMN_FAMILY = 'XYZ'\n"; + " CONSTRAINT pk PRIMARY KEY (a_string)) " + generateDDLOptions("DEFAULT_COLUMN_FAMILY = 'XYZ'"); try { conn.createStatement().execute(ddl); conn.createStatement().execute( @@ -1590,7 +1654,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { conn.setAutoCommit(false); String ddl = "CREATE TABLE " + dataTableFullName + " (a_string varchar not null, col1 integer" + - " CONSTRAINT pk PRIMARY KEY (a_string))\n"; + " CONSTRAINT pk PRIMARY KEY (a_string)) " + tableDDLOptions; try { conn.createStatement().execute(ddl); conn.createStatement().execute("ALTER TABLE " + dataTableFullName + " ADD col2 integer IN_MEMORY=true"); @@ -1618,7 +1682,8 @@ public class AlterTableIT extends ParallelStatsDisabledIT { "CREATE TABLE " + dataTableFullName + " (a_string varchar not null, col1 integer, cf1.col2 integer, col3 integer , cf2.col4 integer " - + " CONSTRAINT pk PRIMARY KEY (a_string)) immutable_rows=true , SALT_BUCKETS=3 "); + + " CONSTRAINT pk PRIMARY KEY (a_string)) " + generateDDLOptions("immutable_rows=true , SALT_BUCKETS=3 " + + (!columnEncoded ? ",IMMUTABLE_STORAGE_SCHEME=" + PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN : ""))); String ddl = "Alter table " + dataTableFullName + " add cf3.col5 integer, cf4.col6 integer in_memory=true"; conn.createStatement().execute(ddl); @@ -1656,7 +1721,8 @@ public class AlterTableIT extends ParallelStatsDisabledIT { "CREATE TABLE " + dataTableFullName + " (a_string varchar not null, col1 integer, cf1.col2 integer, col3 integer , cf2.col4 integer " - + " CONSTRAINT pk PRIMARY KEY (a_string)) immutable_rows=true , SALT_BUCKETS=3 "); + + " CONSTRAINT pk PRIMARY KEY (a_string)) " + generateDDLOptions("immutable_rows=true , SALT_BUCKETS=3 " + + (!columnEncoded ? ",IMMUTABLE_STORAGE_SCHEME=" + PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN : ""))); String ddl = "Alter table " + dataTableFullName + " add cf1.col5 integer in_memory=true"; conn.createStatement().execute(ddl); @@ -1686,7 +1752,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { try { String ddl = "CREATE TABLE " + dataTableFullName - + " (pk char(2) not null primary key, col1 integer, b.col1 integer) SPLIT ON ('EA','EZ')"; + + " (pk char(2) not null primary key, col1 integer, b.col1 integer) " + tableDDLOptions + " SPLIT ON ('EA','EZ') "; conn.createStatement().execute(ddl); ddl = "ALTER TABLE " + dataTableFullName + " add b.col2 varchar ttl=30"; conn.createStatement().execute(ddl); @@ -1706,7 +1772,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { try { String ddl = "CREATE TABLE " + dataTableFullName - + " (pk char(2) not null primary key) TTL=100 SPLIT ON ('EA','EZ')"; + + " (pk char(2) not null primary key) " + generateDDLOptions("TTL=100") + " SPLIT ON ('EA','EZ')"; conn.createStatement().execute(ddl); ddl = "ALTER TABLE " + dataTableFullName + " add col1 varchar ttl=30"; conn.createStatement().execute(ddl); @@ -1736,7 +1802,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { + " col1 integer NOT NULL," + " col2 bigint NOT NULL," + " CONSTRAINT NAME_PK PRIMARY KEY (id, col1, col2)" - + " ) TTL=86400, SALT_BUCKETS = 4, DEFAULT_COLUMN_FAMILY='XYZ'"; + + " ) " + generateDDLOptions("TTL=86400, SALT_BUCKETS = 4, DEFAULT_COLUMN_FAMILY='XYZ'"); conn.createStatement().execute(ddl); try (HBaseAdmin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin()) { HTableDescriptor tableDesc = admin.getTableDescriptor(Bytes.toBytes(dataTableFullName)); @@ -1771,7 +1837,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { + " col1 integer NOT NULL," + " col2 bigint NOT NULL," + " CONSTRAINT NAME_PK PRIMARY KEY (id, col1, col2)" - + " ) TTL=86400, SALT_BUCKETS = 4, DEFAULT_COLUMN_FAMILY='XYZ'"; + + " ) " + generateDDLOptions("TTL=86400, SALT_BUCKETS = 4, DEFAULT_COLUMN_FAMILY='XYZ'"); conn.createStatement().execute(ddl); ddl = "ALTER TABLE " + dataTableFullName + " SET IN_MEMORY=true"; conn.createStatement().execute(ddl); @@ -1799,7 +1865,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { + " col1 integer NOT NULL," + " col2 bigint NOT NULL," + " CONSTRAINT NAME_PK PRIMARY KEY (id, col1, col2)" - + " ) TTL=86400, SALT_BUCKETS = 4"; + + " ) " + generateDDLOptions("TTL=86400, SALT_BUCKETS = 4"); conn.createStatement().execute(ddl); ddl = "ALTER TABLE " + dataTableFullName + " SET IN_MEMORY=true"; conn.createStatement().execute(ddl); @@ -1827,7 +1893,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { + " col1 integer NOT NULL," + " col2 bigint NOT NULL," + " CONSTRAINT NAME_PK PRIMARY KEY (id, col1, col2)" - + " ) TTL=86400, SALT_BUCKETS = 4, DEFAULT_COLUMN_FAMILY='XYZ'"; + + " ) " + generateDDLOptions("TTL=86400, SALT_BUCKETS = 4, DEFAULT_COLUMN_FAMILY='XYZ'"); conn.createStatement().execute(ddl); ddl = "ALTER TABLE " + dataTableFullName + " ADD COL3 INTEGER IN_MEMORY=true"; conn.createStatement().execute(ddl); @@ -1855,7 +1921,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { + " col1 integer NOT NULL," + " col2 bigint NOT NULL," + " CONSTRAINT NAME_PK PRIMARY KEY (id, col1, col2)" - + " ) TTL=86400, SALT_BUCKETS = 4, DEFAULT_COLUMN_FAMILY='XYZ'"; + + " ) " + generateDDLOptions("TTL=86400, SALT_BUCKETS = 4, DEFAULT_COLUMN_FAMILY='XYZ'"); conn.createStatement().execute(ddl); ddl = "ALTER TABLE " + dataTableFullName + " ADD NEWCF.COL3 INTEGER IN_MEMORY=true"; conn.createStatement().execute(ddl); @@ -1885,7 +1951,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { + " col1 integer NOT NULL," + " col2 bigint NOT NULL," + " CONSTRAINT NAME_PK PRIMARY KEY (id, col1, col2)" - + " ) TTL=86400, SALT_BUCKETS = 4, DEFAULT_COLUMN_FAMILY='XYZ'"; + + " ) " + generateDDLOptions("TTL=86400, SALT_BUCKETS = 4, DEFAULT_COLUMN_FAMILY='XYZ'"); conn.createStatement().execute(ddl); ddl = "ALTER TABLE " + dataTableFullName + " ADD NEWCF.COL3 INTEGER IN_MEMORY=true"; conn.createStatement().execute(ddl); @@ -1949,7 +2015,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { + " col1 integer NOT NULL," + " col2 bigint NOT NULL," + " CONSTRAINT NAME_PK PRIMARY KEY (id, col1, col2)" - + " )"; + + " ) " + tableDDLOptions; conn.createStatement().execute(ddl); ddl = "ALTER TABLE " + dataTableFullName + " ADD NEWCF.COL3 INTEGER NEWCF.UNKNOWN_PROP='ABC'"; try { @@ -1975,7 +2041,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { Connection conn = DriverManager.getConnection(getUrl(), props); Statement stmt = conn.createStatement(); - stmt.execute("CREATE TABLE " + dataTableFullName + " (id SMALLINT PRIMARY KEY, name VARCHAR)"); + stmt.execute("CREATE TABLE " + dataTableFullName + " (id SMALLINT PRIMARY KEY, name VARCHAR) "+tableDDLOptions); ResultSet rs = stmt.executeQuery("SELECT STORE_NULLS FROM \"SYSTEM\".\"CATALOG\" " + "WHERE table_name = '" @@ -2007,7 +2073,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { + " k2 integer NOT NULL," + " col1 bigint," + " CONSTRAINT NAME_PK PRIMARY KEY (k1, k2)" - + " )"; + + " ) "+tableDDLOptions; conn.createStatement().execute(ddl); // set HTableProperty when adding a pk column should fail @@ -2081,7 +2147,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { + " col1 integer NOT NULL," + " col2 bigint NOT NULL," + " CONSTRAINT NAME_PK PRIMARY KEY (id, col1, col2)" - + " )"; + + " ) "+tableDDLOptions; conn.createStatement().execute(ddl); asssertIsWALDisabled(conn, dataTableFullName, false); @@ -2118,12 +2184,12 @@ public class AlterTableIT extends ParallelStatsDisabledIT { @Test public void testDeclaringColumnAsRowTimestamp() throws Exception { try (Connection conn = DriverManager.getConnection(getUrl())) { - conn.createStatement().execute("CREATE TABLE " + dataTableFullName + " (PK1 DATE NOT NULL, PK2 VARCHAR NOT NULL, KV1 VARCHAR CONSTRAINT PK PRIMARY KEY(PK1 ROW_TIMESTAMP, PK2)) "); + conn.createStatement().execute("CREATE TABLE " + dataTableFullName + " (PK1 DATE NOT NULL, PK2 VARCHAR NOT NULL, KV1 VARCHAR CONSTRAINT PK PRIMARY KEY(PK1 ROW_TIMESTAMP, PK2)) " + tableDDLOptions); PhoenixConnection phxConn = conn.unwrap(PhoenixConnection.class); PTable table = phxConn.getTable(new PTableKey(phxConn.getTenantId(), dataTableFullName)); // Assert that the column shows up as row time stamp in the cache. - assertTrue(table.getColumn("PK1").isRowTimestamp()); - assertFalse(table.getColumn("PK2").isRowTimestamp()); + assertTrue(table.getColumnForColumnName("PK1").isRowTimestamp()); + assertFalse(table.getColumnForColumnName("PK2").isRowTimestamp()); assertIsRowTimestampSet(schemaName, dataTableName, "PK1"); String dataTableName2 = BaseTest.generateUniqueName(); @@ -2131,18 +2197,17 @@ public class AlterTableIT extends ParallelStatsDisabledIT { conn.createStatement().execute("CREATE TABLE " + dataTableFullName2 + " (PK1 VARCHAR, PK2 DATE PRIMARY KEY ROW_TIMESTAMP, KV1 VARCHAR, KV2 INTEGER)"); table = phxConn.getTable(new PTableKey(phxConn.getTenantId(), dataTableFullName2)); // Assert that the column shows up as row time stamp in the cache. - assertFalse(table.getColumn("PK1").isRowTimestamp()); - assertTrue(table.getColumn("PK2").isRowTimestamp()); + assertFalse(table.getColumnForColumnName("PK1").isRowTimestamp()); + assertTrue(table.getColumnForColumnName("PK2").isRowTimestamp()); assertIsRowTimestampSet(schemaName, dataTableName2, "PK2"); // Create an index on a table has a row time stamp pk column. The column should show up as a row time stamp column for the index too. conn.createStatement().execute("CREATE INDEX " + indexTableName + " ON " + dataTableFullName2 + " (KV1) include (KV2)"); PTable indexTable = phxConn.getTable(new PTableKey(phxConn.getTenantId(), indexTableFullName)); - String indexColName = IndexUtil.getIndexColumnName(table.getColumn("PK2")); + String indexColName = IndexUtil.getIndexColumnName(table.getColumnForColumnName("PK2")); // Assert that the column shows up as row time stamp in the cache. - assertTrue(indexTable.getColumn(indexColName).isRowTimestamp()); + assertTrue(indexTable.getColumnForColumnName(indexColName).isRowTimestamp()); assertIsRowTimestampSet(schemaName, indexTableName, indexColName); - String viewTableName2 = dataTableName2 + "_VIEW"; String viewTableFullName2 = SchemaUtil.getTableName(schemaName, viewTableName2); // Creating a view with a row_timestamp column in its pk constraint is not allowed @@ -2177,7 +2242,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { @Test public void testAddingRowTimestampColumnNotAllowedViaAlterTable() throws Exception { try (Connection conn = DriverManager.getConnection(getUrl())) { - conn.createStatement().execute("CREATE TABLE " + dataTableFullName + " (PK1 VARCHAR NOT NULL, PK2 VARCHAR NOT NULL, KV1 VARCHAR CONSTRAINT PK PRIMARY KEY(PK1, PK2)) "); + conn.createStatement().execute("CREATE TABLE " + dataTableFullName + " (PK1 VARCHAR NOT NULL, PK2 VARCHAR NOT NULL, KV1 VARCHAR CONSTRAINT PK PRIMARY KEY(PK1, PK2)) " + tableDDLOptions); // adding a new pk column that is also row_timestamp is not allowed try { conn.createStatement().execute("ALTER TABLE " + dataTableFullName + " ADD PK3 DATE PRIMARY KEY ROW_TIMESTAMP"); @@ -2195,7 +2260,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { try (Connection conn = DriverManager.getConnection(getUrl(), props)) { // creating a transactional table should fail if transactions are disabled try { - conn.createStatement().execute("CREATE TABLE " + dataTableFullName + "(k INTEGER PRIMARY KEY, v VARCHAR) TRANSACTIONAL=true"); + conn.createStatement().execute("CREATE TABLE " + dataTableFullName + "(k INTEGER PRIMARY KEY, v VARCHAR) " + generateDDLOptions("TRANSACTIONAL=true")); fail(); } catch (SQLException e) { assertEquals(SQLExceptionCode.CANNOT_CREATE_TXN_TABLE_IF_TXNS_DISABLED.getErrorCode(), e.getErrorCode()); @@ -2211,5 +2276,299 @@ public class AlterTableIT extends ParallelStatsDisabledIT { } } + @Test + public void testMetadataForImmutableTable() throws Exception { + String schemaName = "XYZ"; + String baseTableName = generateUniqueName(); + String viewName = generateUniqueName(); + String fullTableName = schemaName + "." + baseTableName; + String fullViewName = schemaName + "." + viewName; + try (Connection conn = DriverManager.getConnection(getUrl())) { + PhoenixConnection phxConn = conn.unwrap(PhoenixConnection.class); + conn.createStatement().execute("CREATE TABLE IF NOT EXISTS " + fullTableName + " (" + + " ID char(1) NOT NULL," + + " COL1 integer NOT NULL," + + " COL2 bigint NOT NULL," + + " KV1 VARCHAR" + + " CONSTRAINT NAME_PK PRIMARY KEY (ID, COL1, COL2)" + + " ) " + generateDDLOptions("IMMUTABLE_ROWS = true" + + (!columnEncoded ? ",IMMUTABLE_STORAGE_SCHEME="+ PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN : ""))); + PTable baseTable = phxConn.getTable(new PTableKey(phxConn.getTenantId(), fullTableName)); + long initBaseTableSeqNumber = baseTable.getSequenceNumber(); + + // assert that the client side cache is updated. + EncodedCQCounter cqCounter = baseTable.getEncodedCQCounter(); + assertEquals( columnEncoded ? (Integer)(ENCODED_CQ_COUNTER_INITIAL_VALUE + 1) : null, cqCounter.getNextQualifier(QueryConstants.DEFAULT_COLUMN_FAMILY)); + + // assert that the server side metadata is updated correctly. + assertEncodedCQCounter(DEFAULT_COLUMN_FAMILY, schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 1); + assertEncodedCQValue(DEFAULT_COLUMN_FAMILY, "KV1", schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE); + assertSequenceNumber(schemaName, baseTableName, initBaseTableSeqNumber); + + // now create a view and validate client and server side metadata + String viewDDL = "CREATE VIEW " + fullViewName + " ( VIEW_COL1 INTEGER, A.VIEW_COL2 VARCHAR ) AS SELECT * FROM " + fullTableName; + conn.createStatement().execute(viewDDL); + baseTable = phxConn.getTable(new PTableKey(phxConn.getTenantId(), fullTableName)); + PTable view = phxConn.getTable(new PTableKey(phxConn.getTenantId(), fullViewName)); + + // verify that the client side cache is updated. Base table's cq counters should be updated. + assertEquals( columnEncoded ? (Integer)(ENCODED_CQ_COUNTER_INITIAL_VALUE + 2) : null, baseTable.getEncodedCQCounter().getNextQualifier(DEFAULT_COLUMN_FAMILY)); + assertEquals( columnEncoded ? (Integer)(ENCODED_CQ_COUNTER_INITIAL_VALUE + 1) : null, baseTable.getEncodedCQCounter().getNextQualifier("A")); + assertNull("A view should always have the null cq counter", view.getEncodedCQCounter().getNextQualifier(DEFAULT_COLUMN_FAMILY)); + + // assert that the server side metadata for the base table and the view is also updated correctly. + assertEncodedCQCounter(DEFAULT_COLUMN_FAMILY, schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 2); + assertEncodedCQCounter("A", schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 1); + assertEncodedCQValue(DEFAULT_COLUMN_FAMILY, "VIEW_COL1", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 1); + assertEncodedCQValue("A", "VIEW_COL2", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE); + assertSequenceNumber(schemaName, baseTableName, initBaseTableSeqNumber + (columnEncoded ? 1 : 0)); + assertSequenceNumber(schemaName, viewName, PTable.INITIAL_SEQ_NUM); + } + } + + @Test + public void testMetadataForMutableTable() throws Exception { + String schemaName = "XYZ"; + String baseTableName = generateUniqueName(); + String viewName = generateUniqueName(); + String fullTableName = schemaName + "." + baseTableName; + String fullViewName = schemaName + "." + viewName; + try (Connection conn = DriverManager.getConnection(getUrl())) { + PhoenixConnection phxConn = conn.unwrap(PhoenixConnection.class); + conn.createStatement().execute("CREATE TABLE IF NOT EXISTS " + fullTableName + " (" + + " ID char(1) NOT NULL," + + " COL1 integer NOT NULL," + + " COL2 bigint NOT NULL," + + " KV1 VARCHAR" + + " CONSTRAINT NAME_PK PRIMARY KEY (ID, COL1, COL2)" + + " ) " + tableDDLOptions); + PTable baseTable = phxConn.getTable(new PTableKey(phxConn.getTenantId(), fullTableName)); + long initBaseTableSeqNumber = baseTable.getSequenceNumber(); + + // assert that the client side cache is updated. + EncodedCQCounter cqCounter = baseTable.getEncodedCQCounter(); + assertEquals( columnEncoded ? (Integer)(ENCODED_CQ_COUNTER_INITIAL_VALUE + 1) : null, cqCounter.getNextQualifier(QueryConstants.DEFAULT_COLUMN_FAMILY)); + + + // assert that the server side metadata is updated correctly. + assertEncodedCQCounter(DEFAULT_COLUMN_FAMILY, schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 1); + assertEncodedCQValue(DEFAULT_COLUMN_FAMILY, "KV1", schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE); + assertSequenceNumber(schemaName, baseTableName, initBaseTableSeqNumber); + + // now create a view and validate client and server side metadata + String viewDDL = "CREATE VIEW " + fullViewName + " ( VIEW_COL1 INTEGER, A.VIEW_COL2 VARCHAR ) AS SELECT * FROM " + fullTableName; + conn.createStatement().execute(viewDDL); + baseTable = phxConn.getTable(new PTableKey(phxConn.getTenantId(), fullTableName)); + PTable view = phxConn.getTable(new PTableKey(phxConn.getTenantId(), fullViewName)); + + // verify that the client side cache is updated. Base table's cq counters should be updated. + assertEquals(columnEncoded ? (Integer)(ENCODED_CQ_COUNTER_INITIAL_VALUE + 3) : null, baseTable.getEncodedCQCounter().getNextQualifier(DEFAULT_COLUMN_FAMILY)); + assertNull("A view should always have the null cq counter", view.getEncodedCQCounter().getNextQualifier(DEFAULT_COLUMN_FAMILY)); + + // assert that the server side metadata for the base table and the view is also updated correctly. + assertEncodedCQCounter(DEFAULT_COLUMN_FAMILY, schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 3); + assertEncodedCQValue(DEFAULT_COLUMN_FAMILY, "VIEW_COL1", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 1); + assertEncodedCQValue("A", "VIEW_COL2", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 2); + assertSequenceNumber(schemaName, baseTableName, initBaseTableSeqNumber + (columnEncoded ? 1 : 0)); + assertSequenceNumber(schemaName, viewName, PTable.INITIAL_SEQ_NUM); + } + } + + @Test + public void testAddingColumnsToTablesAndViews() throws Exception { + String schemaName = generateUniqueName(); + String baseTableName = generateUniqueName(); + String viewName = generateUniqueName(); + String fullTableName = schemaName + "." + baseTableName; + String fullViewName = schemaName + "." + viewName; + Properties props = new Properties(); + props.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, Boolean.toString(true)); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + conn.createStatement().execute("CREATE SCHEMA " + schemaName); + PhoenixConnection phxConn = conn.unwrap(PhoenixConnection.class); + conn.createStatement().execute("CREATE TABLE " + fullTableName + " (" + + " ID char(1) NOT NULL," + + " COL1 integer NOT NULL," + + " COL2 bigint NOT NULL," + + " CONSTRAINT NAME_PK PRIMARY KEY (ID, COL1, COL2)" + + " ) " + tableDDLOptions); + PTable baseTable = phxConn.getTable(new PTableKey(phxConn.getTenantId(), fullTableName)); + long initBaseTableSeqNumber = baseTable.getSequenceNumber(); + + // Add a column to the base table and see if the client and server metadata is updated correctly + String alterDDL = "ALTER TABLE " + fullTableName + " ADD COL3 VARCHAR PRIMARY KEY, COL4 INTEGER, COL5 VARCHAR, B.COL6 DECIMAL (10, 2)"; + conn.createStatement().execute(alterDDL); + + // assert that the client side cache is updated. + baseTable = phxConn.getTable(new PTableKey(phxConn.getTenantId(), fullTableName)); + EncodedCQCounter encodedCqCounter = baseTable.getEncodedCQCounter(); + assertEquals( columnEncoded ?(Integer)(ENCODED_CQ_COUNTER_INITIAL_VALUE + 3) : null, encodedCqCounter.getNextQualifier(DEFAULT_COLUMN_FAMILY)); + + // assert that the server side metadata is updated correctly. + assertEncodedCQCounter(DEFAULT_COLUMN_FAMILY, schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 3); + + // assert that the server side metadata for columns is updated correctly. + assertEncodedCQValue(DEFAULT_COLUMN_FAMILY, "COL4", schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE); + assertEncodedCQValue(DEFAULT_COLUMN_FAMILY, "COL5", schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 1); + assertEncodedCQValue("B", "COL6", schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 2); + long baseTableSeqNumBeforeAddingChildCols = initBaseTableSeqNumber + 1; + assertSequenceNumber(schemaName, baseTableName, baseTableSeqNumBeforeAddingChildCols); + + // Create a view + String viewDDL = "CREATE VIEW " + fullViewName + " ( VIEW_COL1 INTEGER, A.VIEW_COL2 VARCHAR ) AS SELECT * FROM " + fullTableName; + conn.createStatement().execute(viewDDL); + + // assert that the server side metadata is updated correctly. + assertEncodedCQCounter(DEFAULT_COLUMN_FAMILY, schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 5); + + // assert that the server side metadata for columns is updated correctly. + assertEncodedCQValue(DEFAULT_COLUMN_FAMILY, "VIEW_COL1", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 3); + assertEncodedCQValue("A", "VIEW_COL2", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 4); + // for encoded columns creating a view that adds its own columns should increment the base table's sequence number too. + assertSequenceNumber(schemaName, baseTableName, columnEncoded ? initBaseTableSeqNumber + 2 : baseTableSeqNumBeforeAddingChildCols ); + + // Add column to the view + viewDDL = "ALTER VIEW " + fullViewName + " ADD VIEW_COL3 DECIMAL(10, 2), A.VIEW_COL4 VARCHAR, B.VIEW_COL5 INTEGER"; + conn.createStatement().execute(viewDDL); + + // assert that the client cache for the base table is updated + baseTable = phxConn.getTable(new PTableKey(phxConn.getTenantId(), fullTableName)); + encodedCqCounter = baseTable.getEncodedCQCounter(); + assertEquals( columnEncoded ? (Integer)(ENCODED_CQ_COUNTER_INITIAL_VALUE + 8) : null, encodedCqCounter.getNextQualifier(DEFAULT_COLUMN_FAMILY)); + + // assert client cache for view + PTable view = phxConn.getTable(new PTableKey(phxConn.getTenantId(), fullViewName)); + encodedCqCounter = view.getEncodedCQCounter(); + assertNull("A view should always have the column qualifier counter as null", view.getEncodedCQCounter().getNextQualifier(DEFAULT_COLUMN_FAMILY)); + + // assert that the server side metadata for the base table and the view is also updated correctly. + assertEncodedCQCounter(DEFAULT_COLUMN_FAMILY, schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 8); + assertEncodedCQValue(DEFAULT_COLUMN_FAMILY, "VIEW_COL1", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 3); + assertEncodedCQValue("A", "VIEW_COL2", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 4); + assertEncodedCQValue(DEFAULT_COLUMN_FAMILY, "VIEW_COL3", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 5); + assertEncodedCQValue("A", "VIEW_COL4", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 6); + assertEncodedCQValue("B", "VIEW_COL5", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 7); + // adding a column to the should increment the base table's sequence number too since we update the cq counters for column families. + assertSequenceNumber(schemaName, baseTableName, columnEncoded ? initBaseTableSeqNumber + 3 : baseTableSeqNumBeforeAddingChildCols ); + assertSequenceNumber(schemaName, viewName, PTable.INITIAL_SEQ_NUM + 1); + + // Add column to the base table which doesn't already exist in the view. + alterDDL = "ALTER TABLE " + fullTableName + " ADD COL10 VARCHAR, A.COL11 INTEGER"; + conn.createStatement().execute(alterDDL); + baseTable = phxConn.getTable(new PTableKey(phxConn.getTenantId(), fullTableName)); + + // assert that the client cache for the base table is updated + encodedCqCounter = baseTable.getEncodedCQCounter(); + assertEquals( columnEncoded ? (Integer)(ENCODED_CQ_COUNTER_INITIAL_VALUE + 10) : null, encodedCqCounter.getNextQualifier(DEFAULT_COLUMN_FAMILY)); + + // assert client cache for view + view = phxConn.getTable(new PTableKey(phxConn.getTenantId(), fullViewName)); + encodedCqCounter = view.getEncodedCQCounter(); + assertNull("A view should always have the column qualifier counter as null", view.getEncodedCQCounter().getNextQualifier(DEFAULT_COLUMN_FAMILY)); + + // assert that the server side metadata for the base table and the view is also updated correctly. + assertEncodedCQCounter(DEFAULT_COLUMN_FAMILY, schemaName, baseTableName, (ENCODED_CQ_COUNTER_INITIAL_VALUE + 10)); + assertEncodedCQValue(DEFAULT_COLUMN_FAMILY, "COL10", schemaName, viewName, (ENCODED_CQ_COUNTER_INITIAL_VALUE + 8)); + assertEncodedCQValue("A", "COL11", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 9); + assertSequenceNumber(schemaName, baseTableName, columnEncoded ? initBaseTableSeqNumber + 4 : initBaseTableSeqNumber + 2 ); + assertSequenceNumber(schemaName, viewName, PTable.INITIAL_SEQ_NUM + 2); + } + } + + private void assertEncodedCQValue(String columnFamily, String columnName, String schemaName, String tableName, int expectedValue) throws Exception { + String query = "SELECT " + COLUMN_QUALIFIER + " FROM SYSTEM.CATALOG WHERE " + TABLE_SCHEM + " = ? AND " + TABLE_NAME + + " = ? " + " AND " + COLUMN_FAMILY + " = ?" + " AND " + COLUMN_NAME + " = ?" + " AND " + COLUMN_QUALIFIER + " IS NOT NULL"; + try (Connection conn = DriverManager.getConnection(getUrl())) { + PreparedStatement stmt = conn.prepareStatement(query); + stmt.setString(1, schemaName); + stmt.setString(2, tableName); + stmt.setString(3, columnFamily); + stmt.setString(4, columnName); + ResultSet rs = stmt.executeQuery(); + assertTrue(rs.next()); + if (columnEncoded) { + assertTrue(Bytes.equals(QualifierEncodingScheme.TWO_BYTE_QUALIFIERS.encode(expectedValue), rs.getBytes(1))); + } else { + assertTrue(Bytes.equals(columnName.getBytes(), rs.getBytes(1))); + } + assertFalse(rs.next()); + } + } + + private void assertEncodedCQCounter(String columnFamily, String schemaName, String tableName, int expectedValue) throws Exception { + String query = "SELECT " + COLUMN_QUALIFIER_COUNTER + " FROM SYSTEM.CATALOG WHERE " + TABLE_SCHEM + " = ? AND " + TABLE_NAME + + " = ? " + " AND " + COLUMN_FAMILY + " = ? AND " + COLUMN_QUALIFIER_COUNTER + " IS NOT NULL"; + try (Connection conn = DriverManager.getConnection(getUrl())) { + PreparedStatement stmt = conn.prepareStatement(query); + stmt.setString(1, schemaName); + stmt.setString(2, tableName); + stmt.setString(3, columnFamily); + ResultSet rs = stmt.executeQuery(); + if (columnEncoded) { + assertTrue(rs.next()); + assertEquals(expectedValue, rs.getInt(1)); + assertFalse(rs.next()); + } else { + assertFalse(rs.next()); + } + } + } + + private void assertSequenceNumber(String schemaName, String tableName, long expectedSequenceNumber) throws Exception { + String query = "SELECT " + TABLE_SEQ_NUM + " FROM SYSTEM.CATALOG WHERE " + TABLE_SCHEM + " = ? AND " + TABLE_NAME + + " = ? AND " + TABLE_SEQ_NUM + " IS NOT NULL AND " + COLUMN_NAME + " IS NULL AND " + + COLUMN_FAMILY + " IS NULL "; + try (Connection conn = DriverManager.getConnection(getUrl())) { + PreparedStatement stmt = conn.prepareStatement(query); + stmt.setString(1, schemaName); + stmt.setString(2, tableName); + ResultSet rs = stmt.executeQuery(); + assertTrue(rs.next()); + assertEquals(expectedSequenceNumber, rs.getInt(1)); + assertFalse(rs.next()); + } + } + + @Test + public void testAlterImmutableRowsPropertyForOneCellPerKeyValueColumnStorageScheme() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + String ddl = "CREATE TABLE " + dataTableFullName + " (\n" + +"ID VARCHAR(15) NOT NULL,\n" + +"CREATED_DATE DATE,\n" + +"CREATION_TIME BIGINT,\n" + +"CONSTRAINT PK PRIMARY KEY (ID)) " + tableDDLOptions; + Connection conn = DriverManager.getConnection(getUrl(), props); + conn.createStatement().execute(ddl); + assertImmutableRows(conn, dataTableFullName, false); + ddl = "ALTER TABLE " + dataTableFullName + " SET IMMUTABLE_ROWS = true"; + conn.createStatement().execute(ddl); + assertImmutableRows(conn, dataTableFullName, true); + } + + @Test + public void testAlterImmutableRowsPropertyForOneCellPerColumnFamilyStorageScheme() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + String ddl = "CREATE TABLE " + dataTableFullName + " (\n" + +"ID VARCHAR(15) NOT NULL,\n" + +"CREATED_DATE DATE,\n" + +"CREATION_TIME BIGINT,\n" + +"CONSTRAINT PK PRIMARY KEY (ID)) " + generateDDLOptions("COLUMN_ENCODED_BYTES=4, IMMUTABLE_ROWS=true" + + (!columnEncoded ? ",IMMUTABLE_STORAGE_SCHEME=" + PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN : "")); + Connection conn = DriverManager.getConnection(getUrl(), props); + conn.createStatement().execute(ddl); + assertImmutableRows(conn, dataTableFullName, true); + try { + ddl = "ALTER TABLE " + dataTableFullName + " SET IMMUTABLE_ROWS = false"; + conn.createStatement().execute(ddl); + if (columnEncoded) { + fail(); + } + } + catch(SQLException e) { + assertEquals(SQLExceptionCode.CANNOT_ALTER_IMMUTABLE_ROWS_PROPERTY.getErrorCode(), e.getErrorCode()); + } + assertImmutableRows(conn, dataTableFullName, columnEncoded); + } + }
