This is an automated email from the ASF dual-hosted git repository. lokiore pushed a commit to branch PHOENIX-7210-feature in repository https://gitbox.apache.org/repos/asf/phoenix.git
commit c912d7b3c24e0555f09141488c675b28e029b17d Merge: 11e0996795 1f88ca83f0 Author: Lokesh Khurana <lokesh.khur...@salesforce.com> AuthorDate: Thu Jul 25 11:48:38 2024 -0700 Merge branch 'master' of github.com:apache/phoenix into PHOENIX-7210-feature .gitignore | 3 + Jenkinsfile | 2 +- phoenix-assembly/pom.xml | 5 + phoenix-core-client/pom.xml | 15 + phoenix-core-client/src/main/antlr3/PhoenixSQL.g | 41 + ...java => InvalidateMetadataCacheController.java} | 36 +- ... InvalidateMetadataCacheControllerFactory.java} | 26 +- .../controller/ServerSideRPCControllerFactory.java | 3 - .../apache/phoenix/cache/ServerCacheClient.java | 16 +- .../apache/phoenix/cache/ServerMetadataCache.java | 28 + .../phoenix/cache/ServerMetadataCacheImpl.java | 156 ++ .../phoenix/compile/CreateIndexCompiler.java | 9 +- .../phoenix/compile/CreateTableCompiler.java | 30 +- .../apache/phoenix/compile/ExpressionCompiler.java | 44 +- .../org/apache/phoenix/compile/FromCompiler.java | 15 +- .../apache/phoenix/compile/GroupByCompiler.java | 5 + .../phoenix/compile/OrderPreservingTracker.java | 246 +- .../apache/phoenix/compile/ProjectionCompiler.java | 274 +- .../org/apache/phoenix/compile/QueryCompiler.java | 72 + .../org/apache/phoenix/compile/ScanRanges.java | 3 +- .../apache/phoenix/compile/StatementContext.java | 29 +- .../phoenix/compile/TupleProjectionCompiler.java | 17 +- .../org/apache/phoenix/compile/UpsertCompiler.java | 196 +- .../org/apache/phoenix/compile/WhereOptimizer.java | 183 +- .../BaseScannerRegionObserverConstants.java | 10 +- .../InvalidateServerMetadataCacheRequest.java | 53 + .../coprocessorclient/MetaDataProtocol.java | 2 +- .../phoenix/coprocessorclient/RowKeyMatcher.java | 150 ++ .../phoenix/coprocessorclient/TableTTLInfo.java | 115 + .../coprocessorclient/TableTTLInfoCache.java | 108 + .../metrics/MetricsMetadataCachingSource.java | 230 ++ .../metrics/MetricsMetadataCachingSourceImpl.java | 122 + .../MetricsPhoenixCoprocessorSourceFactory.java | 12 + .../apache/phoenix/exception/SQLExceptionCode.java | 45 +- .../apache/phoenix/exception/SQLExceptionInfo.java | 33 + .../StaleMetadataCacheException.java} | 23 +- .../org/apache/phoenix/execute/BaseQueryPlan.java | 28 +- .../org/apache/phoenix/execute/MutationState.java | 174 +- .../execute/PhoenixTxIndexMutationGenerator.java | 2 +- .../apache/phoenix/execute/SortMergeJoinPlan.java | 15 +- .../phoenix/execute/TupleProjectionPlan.java | 21 +- .../org/apache/phoenix/execute/TupleProjector.java | 4 +- .../apache/phoenix/expression/ExpressionType.java | 6 +- .../expression/SingleCellColumnExpression.java | 17 +- .../function/DistinctCountAggregateFunction.java | 6 + .../expression/function/JsonExistsFunction.java | 110 + .../expression/function/JsonModifyFunction.java | 120 + .../expression/function/JsonQueryFunction.java | 120 + .../expression/function/JsonValueFunction.java | 130 + .../org/apache/phoenix/filter/PagingFilter.java | 109 +- .../phoenix/filter/RowKeyComparisonFilter.java | 2 +- .../org/apache/phoenix/filter/SkipScanFilter.java | 55 +- .../hbase/index/util/ImmutableBytesPtr.java | 37 +- .../hbase/index/util/IndexManagementUtil.java | 13 +- .../org/apache/phoenix/index/CDCTableInfo.java | 276 ++ .../org/apache/phoenix/index/IndexMaintainer.java | 33 +- .../phoenix/iterate/BaseResultIterators.java | 11 +- .../iterate/DefaultParallelScanGrouper.java | 6 +- .../org/apache/phoenix/iterate/ExplainTable.java | 12 +- .../phoenix/iterate/TableResultIterator.java | 2 +- .../org/apache/phoenix/jdbc/PhoenixConnection.java | 18 + .../phoenix/jdbc/PhoenixDatabaseMetaData.java | 19 +- .../org/apache/phoenix/jdbc/PhoenixResultSet.java | 3 +- .../org/apache/phoenix/jdbc/PhoenixStatement.java | 254 +- .../phoenix/monitoring/GlobalClientMetrics.java | 4 +- .../org/apache/phoenix/monitoring/MetricType.java | 3 + .../apache/phoenix/optimize/QueryOptimizer.java | 137 +- .../java/org/apache/phoenix/parse/ColumnDef.java | 17 +- .../java/org/apache/phoenix/parse/ColumnName.java | 6 +- .../apache/phoenix/parse/ComparisonParseNode.java | 2 +- .../apache/phoenix/parse/CreateCDCStatement.java | 71 + ...erminalParseNode.java => DropCDCStatement.java} | 45 +- .../phoenix/parse/FamilyWildcardParseNode.java | 11 +- .../java/org/apache/phoenix/parse/HintNode.java | 177 +- .../apache/phoenix/parse/JsonExistsParseNode.java | 45 + .../apache/phoenix/parse/JsonModifyParseNode.java | 48 + .../apache/phoenix/parse/JsonQueryParseNode.java | 48 + .../apache/phoenix/parse/JsonValueParseNode.java | 45 + .../org/apache/phoenix/parse/NamedTableNode.java | 2 +- .../java/org/apache/phoenix/parse/ParseNode.java | 23 + .../org/apache/phoenix/parse/ParseNodeFactory.java | 18 + .../apache/phoenix/parse/ParseNodeRewriter.java | 2 +- .../org/apache/phoenix/parse/SelectStatement.java | 15 +- .../phoenix/parse/TableWildcardParseNode.java | 10 + .../apache/phoenix/parse/TerminalParseNode.java | 8 + .../apache/phoenix/parse/WildcardParseNode.java | 13 +- .../phoenix/query/ConnectionQueryServices.java | 52 +- .../phoenix/query/ConnectionQueryServicesImpl.java | 475 +++- .../query/ConnectionlessQueryServicesImpl.java | 46 +- .../query/DelegateConnectionQueryServices.java | 42 +- .../org/apache/phoenix/query/QueryConstants.java | 16 + .../org/apache/phoenix/query/QueryServices.java | 28 +- .../apache/phoenix/query/QueryServicesOptions.java | 29 +- .../java/org/apache/phoenix/schema/ColumnRef.java | 3 +- .../ComparisonNotSupportedException.java} | 24 +- .../org/apache/phoenix/schema/DelegateTable.java | 29 +- .../org/apache/phoenix/schema/KeyValueSchema.java | 26 + .../org/apache/phoenix/schema/MetaDataClient.java | 1030 ++++++-- .../java/org/apache/phoenix/schema/PTable.java | 49 +- .../java/org/apache/phoenix/schema/PTableImpl.java | 233 +- .../java/org/apache/phoenix/schema/PTableKey.java | 13 + .../java/org/apache/phoenix/schema/PTableType.java | 4 +- .../org/apache/phoenix/schema/SaltingUtil.java | 3 - .../org/apache/phoenix/schema/TableProperty.java | 32 +- .../java/org/apache/phoenix/schema/TableRef.java | 2 +- .../java/org/apache/phoenix/schema/task/Task.java | 23 +- .../schema/tool/SchemaExtractionProcessor.java | 20 +- .../phoenix/schema/transform/TransformClient.java | 2 - .../org/apache/phoenix/schema/types/PDataType.java | 33 + .../phoenix/schema/types/PDataTypeFactory.java | 1 + .../org/apache/phoenix/schema/types/PJson.java | 130 + .../apache/phoenix/schema/types/PVarbinary.java | 4 + .../org/apache/phoenix/schema/types/PVarchar.java | 5 + .../apache/phoenix/trace/PhoenixMetricsSink.java | 8 +- .../java/org/apache/phoenix/trace/TraceReader.java | 146 +- .../java/org/apache/phoenix/trace/TraceWriter.java | 3 +- .../phoenix/transaction/OmidTransactionTable.java | 26 - .../java/org/apache/phoenix/util/ByteUtil.java | 17 + .../org/apache/phoenix/util/CDCChangeBuilder.java | 151 ++ .../main/java/org/apache/phoenix/util/CDCUtil.java | 141 + .../java/org/apache/phoenix/util/ClientUtil.java | 16 +- .../apache/phoenix/util/EncodedColumnsUtil.java | 8 +- .../java/org/apache/phoenix/util/IndexUtil.java | 9 +- .../apache/phoenix/util/PhoenixKeyValueUtil.java | 51 +- .../org/apache/phoenix/util/PhoenixRuntime.java | 2 + .../java/org/apache/phoenix/util/ScanUtil.java | 155 +- .../java/org/apache/phoenix/util/SchemaUtil.java | 29 +- .../java/org/apache/phoenix/util/UpgradeUtil.java | 227 ++ .../phoenix/util/ValidateLastDDLTimestampUtil.java | 228 ++ .../java/org/apache/phoenix/util/ViewUtil.java | 157 +- .../apache/phoenix/util/json/BsonDataFormat.java | 202 ++ .../apache/phoenix/util/json/BsonJsonProvider.java | 251 ++ .../apache/phoenix/util/json/JsonDataFormat.java | 86 + .../phoenix/util/json/JsonDataFormatFactory.java | 16 +- .../src/main/protobuf/CDCInfo.proto | 33 +- phoenix-core-client/src/main/protobuf/PTable.proto | 4 + .../protobuf/RegionServerEndpointService.proto | 59 + .../src/main/protobuf/ServerCachingService.proto | 1 + phoenix-core-server/pom.xml | 2 +- .../hadoop/hbase/ipc/PhoenixRpcScheduler.java | 37 +- .../hbase/ipc/PhoenixRpcSchedulerFactory.java | 10 +- .../regionserver/IndexHalfStoreFileReader.java | 9 +- .../regionserver/LocalIndexStoreFileScanner.java | 9 +- .../hbase/regionserver/wal/IndexedHLogReader.java | 10 +- .../coprocessor/BaseScannerRegionObserver.java | 23 +- .../coprocessor/CDCGlobalIndexRegionScanner.java | 267 ++ .../phoenix/coprocessor/CompactionScanner.java | 2002 ++++++++++++-- .../coprocessor/GlobalIndexRegionScanner.java | 59 +- .../coprocessor/IndexRebuildRegionScanner.java | 7 +- .../coprocessor/IndexRepairRegionScanner.java | 9 +- .../phoenix/coprocessor/IndexerRegionScanner.java | 6 +- .../phoenix/coprocessor/MetaDataEndpointImpl.java | 471 +++- .../phoenix/coprocessor/PagingRegionScanner.java | 260 +- .../coprocessor/PhoenixAccessController.java | 11 +- .../coprocessor/PhoenixRegionServerEndpoint.java | 117 + .../coprocessor/PhoenixTTLRegionObserver.java | 6 +- .../phoenix/coprocessor/TTLRegionScanner.java | 24 +- .../coprocessor/UncoveredIndexRegionScanner.java | 18 +- .../UngroupedAggregateRegionObserver.java | 215 +- .../coprocessor/VerifyLastDDLTimestamp.java | 69 + .../phoenix/hbase/index/IndexRegionObserver.java | 496 ++-- .../apache/phoenix/hbase/index/LockManager.java | 212 +- .../hbase/index/covered/data/CachedLocalTable.java | 2 +- .../iterate/MapReduceParallelScanGrouper.java | 6 +- .../iterate/NonAggregateRegionScannerFactory.java | 100 +- .../phoenix/iterate/RegionScannerFactory.java | 66 +- .../mapreduce/PhoenixTTLDeleteJobMapper.java | 6 +- .../mapreduce/index/IndexScrutinyMapper.java | 32 +- .../index/automation/PhoenixMRJobSubmitter.java | 27 +- .../PhoenixTransformWithViewsInputFormat.java | 1 + .../util/DefaultPhoenixMultiViewListProvider.java | 2 +- .../mapreduce/util/PhoenixMultiInputUtil.java | 16 +- .../phoenix/mapreduce/util/ViewInfoTracker.java | 2 +- .../apache/phoenix/schema/transform/Transform.java | 6 + .../java/org/apache/phoenix/util/ServerUtil.java | 2 +- .../org/apache/phoenix/util/ServerViewUtil.java | 2 +- phoenix-core/pom.xml | 24 + .../phoenix/cache/ServerMetadataCacheIT.java | 1921 ++++++++++++++ .../phoenix/end2end/AlterTableWithViewsIT.java | 19 +- .../apache/phoenix/end2end/AppendOnlySchemaIT.java | 2 +- .../phoenix/end2end/BackwardCompatibilityIT.java | 1 + .../phoenix/end2end/BaseRowKeyMatcherTestIT.java | 1056 ++++++++ .../org/apache/phoenix/end2end/BaseViewTTLIT.java | 2732 ++++++++++++++++++++ .../java/org/apache/phoenix/end2end/CDCBaseIT.java | 828 ++++++ .../apache/phoenix/end2end/CDCDefinitionIT.java | 321 +++ .../org/apache/phoenix/end2end/CDCQueryIT.java | 712 +++++ .../end2end/ConcurrentMutationsExtendedIT.java | 20 +- .../org/apache/phoenix/end2end/ConnectionIT.java | 3 + .../apache/phoenix/end2end/ConnectionUtilIT.java | 3 + .../org/apache/phoenix/end2end/CreateTableIT.java | 153 +- .../DefaultPhoenixMultiViewListProviderIT.java | 14 +- .../FailingPhoenixRegionServerEndpoint.java | 91 + .../phoenix/end2end/FlappingAlterTableIT.java | 23 +- .../phoenix/end2end/FlappingLocalIndexIT.java | 5 +- .../java/org/apache/phoenix/end2end/InListIT.java | 38 +- .../end2end/IndexRepairRegionScannerIT.java | 1 + .../end2end/IndexScrutinyWithMaxLookbackIT.java | 1 + .../phoenix/end2end/InvalidateMetadataCacheIT.java | 166 ++ .../phoenix/end2end/LoadSystemTableSnapshotIT.java | 3 +- .../phoenix/end2end/LogicalTableNameBaseIT.java | 10 +- .../end2end/LogicalTableNameExtendedIT.java | 21 +- .../LongViewIndexDisabledBaseRowKeyMatcherIT.java | 71 + .../LongViewIndexEnabledBaseRowKeyMatcherIT.java | 71 + .../org/apache/phoenix/end2end/MapReduceIT.java | 8 +- .../phoenix/end2end/MaxLookbackExtendedIT.java | 110 +- .../org/apache/phoenix/end2end/MaxLookbackIT.java | 16 +- .../end2end/MetadataGetTableReadLockIT.java | 150 ++ .../MigrateSystemTablesToSystemNamespaceIT.java | 11 + .../phoenix/end2end/MoveTTLDuringUpgradeIT.java | 152 ++ .../apache/phoenix/end2end/OnDuplicateKey2IT.java | 646 +++++ .../apache/phoenix/end2end/OnDuplicateKeyIT.java | 27 +- .../java/org/apache/phoenix/end2end/OrderByIT.java | 136 + .../PartialResultServerConfigurationIT.java | 3 + .../end2end/PhoenixRegionServerEndpointIT.java | 193 ++ .../PhoenixRegionServerEndpointTestImpl.java | 45 + .../apache/phoenix/end2end/PhoenixTTLToolIT.java | 48 +- .../apache/phoenix/end2end/PropertiesInSyncIT.java | 25 +- .../phoenix/end2end/ReadOnlyViewOnReadOnlyIT.java | 3 +- .../end2end/ServerMetadataCacheTestImpl.java | 89 + .../org/apache/phoenix/end2end/SetPropertyIT.java | 79 + .../end2end/SetPropertyOnEncodedTableIT.java | 2 +- .../end2end/SetPropertyOnNonEncodedTableIT.java | 2 +- .../end2end/SkipScanAfterManualSplitIT.java | 10 +- .../phoenix/end2end/SplitSystemCatalogIT.java | 1 - .../SystemTablesCreationOnConnectionIT.java | 15 + .../apache/phoenix/end2end/TTLAsPhoenixTTLIT.java | 514 ++++ .../it/java/org/apache/phoenix/end2end/TTLIT.java | 853 ++++++ .../org/apache/phoenix/end2end/TableTTLIT.java | 103 +- .../phoenix/end2end/TenantSpecificTablesDMLIT.java | 58 + .../UpdateCacheAcrossDifferentClientsIT.java | 17 +- .../java/org/apache/phoenix/end2end/UpgradeIT.java | 92 +- .../org/apache/phoenix/end2end/ViewMetadataIT.java | 117 + .../java/org/apache/phoenix/end2end/ViewTTLIT.java | 1589 +++--------- .../phoenix/end2end/ViewTTLNotEnabledIT.java | 136 +- .../end2end/ViewTTLWithLongViewIndexEnabledIT.java | 103 + .../end2end/index/DropIndexDuringUpsertIT.java | 2 + .../end2end/index/IndexAsyncThresholdIT.java | 6 +- .../index/IndexVerificationOutputRepositoryIT.java | 2 +- .../index/IndexVerificationResultRepositoryIT.java | 2 +- .../end2end/index/MutableIndexFailureIT.java | 3 + .../phoenix/end2end/index/PartialIndexIT.java | 233 +- .../index/ReplicationWithWALAnnotationIT.java | 11 +- .../phoenix/end2end/index/SingleCellIndexIT.java | 11 +- .../index/UncoveredGlobalIndexRegionScannerIT.java | 7 +- .../phoenix/end2end/json/JsonFunctionsIT.java | 1040 ++++++++ .../phoenix/end2end/transform/TransformToolIT.java | 6 + .../execute/UpsertSelectOverlappingBatchesIT.java | 2 + .../index/FailForUnsupportedHBaseVersionsIT.java | 2 + .../phoenix/iterate/PhoenixQueryTimeoutIT.java | 21 +- .../iterate/RoundRobinResultIteratorIT.java | 10 +- .../phoenix/iterate/ScannerLeaseRenewalIT.java | 2 + .../jdbc/HighAvailabilityTestingUtility.java | 9 + .../jdbc/LoggingSingleConnectionLimiterIT.java | 2 +- .../phoenix/jdbc/ParallelPhoenixConnectionIT.java | 1 + .../monitoring/PhoenixTableLevelMetricsIT.java | 102 +- .../ConnectionQueryServicesMetricsIT.java | 3 + .../phoenix/query/MaxConcurrentConnectionsIT.java | 10 +- .../apache/phoenix/query/MetaDataCachingIT.java | 19 +- .../rpc/UpdateCacheConnectionLevelPropIT.java | 10 + .../java/org/apache/phoenix/rpc/UpdateCacheIT.java | 31 +- .../phoenix/schema/stats/BaseStatsCollectorIT.java | 9 +- .../it/resources/compatible_client_versions.json | 20 +- .../src/it/resources/json/json_datatypes.json | 36 + .../it/resources/json/json_functions_basic.json | 83 + ...dulerTest.java => PhoenixRpcSchedulerTest.java} | 61 +- .../phoenix/cache/ServerCacheClientTest.java | 7 +- .../apache/phoenix/compile/QueryCompilerTest.java | 309 ++- .../apache/phoenix/compile/QueryOptimizerTest.java | 2 + .../apache/phoenix/compile/ViewCompilerTest.java | 3 + .../apache/phoenix/compile/WhereCompilerTest.java | 36 +- .../apache/phoenix/compile/WhereOptimizerTest.java | 87 +- .../phoenix/expression/CoerceExpressionTest.java | 32 +- .../filter/SkipScanFilterIntersectTest.java | 2 +- .../apache/phoenix/filter/SkipScanFilterTest.java | 4 +- .../hbase/index/write/TestWALRecoveryCaching.java | 2 + .../jdbc/ParallelPhoenixConnectionFailureTest.java | 7 +- .../phoenix/jdbc/PhoenixResultSetMetadataTest.java | 2 +- .../mapreduce/PhoenixMultiViewReaderTest.java | 4 +- .../org/apache/phoenix/parse/QueryParserTest.java | 122 +- .../phoenix/query/BaseConnectionlessQueryTest.java | 5 +- .../java/org/apache/phoenix/query/BaseTest.java | 22 + .../query/ConnectionQueryServicesImplTest.java | 45 +- .../phoenix/query/ParallelIteratorsSplitTest.java | 9 +- .../apache/phoenix/query/PhoenixTestBuilder.java | 127 +- .../apache/phoenix/schema/types/PDataTypeTest.java | 1 + .../java/org/apache/phoenix/util/CDCUtilTest.java | 62 + .../org/apache/phoenix/util/IndexUtilTest.java | 4 + .../apache/phoenix/util/PhoenixRuntimeTest.java | 4 + .../org/apache/phoenix/util/RowKeyMatcherTest.java | 287 ++ .../java/org/apache/phoenix/util/ScanUtilTest.java | 2 +- .../java/org/apache/phoenix/util/TestUtil.java | 25 +- .../phoenix/compat/hbase/CompatDelegateHTable.java | 13 +- .../hbase/CompatIndexHalfStoreFileReader.java | 40 + .../compat/hbase/CompatIndexedHLogReader.java | 31 + .../hbase/CompatLocalIndexStoreFileScanner.java | 33 + .../compat/hbase/CompatOmidTransactionTable.java | 42 +- .../compat/hbase/CompatPhoenixRpcScheduler.java | 13 +- .../apache/phoenix/compat/hbase/CompatUtil.java | 15 +- .../compat/hbase/HbaseCompatCapabilities.java | 13 +- .../hbase/ReplicationSinkCompatEndpoint.java | 21 +- .../phoenix/compat/hbase/CompatDelegateHTable.java | 13 +- .../hbase/CompatIndexHalfStoreFileReader.java | 40 + .../compat/hbase/CompatIndexedHLogReader.java | 31 + .../hbase/CompatLocalIndexStoreFileScanner.java | 34 + .../compat/hbase/CompatOmidTransactionTable.java | 42 +- .../compat/hbase/CompatPhoenixRpcScheduler.java | 13 +- .../apache/phoenix/compat/hbase/CompatUtil.java | 15 +- .../compat/hbase/HbaseCompatCapabilities.java | 13 +- .../hbase/ReplicationSinkCompatEndpoint.java | 21 +- .../phoenix/compat/hbase/CompatDelegateHTable.java | 13 +- .../hbase/CompatIndexHalfStoreFileReader.java | 40 + .../compat/hbase/CompatIndexedHLogReader.java | 32 + .../hbase/CompatLocalIndexStoreFileScanner.java | 33 + .../compat/hbase/CompatOmidTransactionTable.java | 42 +- .../compat/hbase/CompatPhoenixRpcScheduler.java | 13 +- .../apache/phoenix/compat/hbase/CompatUtil.java | 15 +- .../compat/hbase/HbaseCompatCapabilities.java | 13 +- .../hbase/ReplicationSinkCompatEndpoint.java | 21 +- phoenix-hbase-compat-2.6.0/pom.xml | 113 + .../phoenix/compat/hbase/CompatDelegateHTable.java | 13 +- .../hbase/CompatIndexHalfStoreFileReader.java | 42 + .../compat/hbase/CompatIndexedHLogReader.java | 23 + .../hbase/CompatLocalIndexStoreFileScanner.java | 35 + .../compat/hbase/CompatOmidTransactionTable.java | 78 + .../compat/hbase/CompatPhoenixRpcScheduler.java | 27 +- .../apache/phoenix/compat/hbase/CompatUtil.java | 15 +- .../compat/hbase/HbaseCompatCapabilities.java | 13 +- .../hbase/ReplicationSinkCompatEndpoint.java | 21 +- phoenix-mapreduce-byo-shaded-hbase/pom.xml | 12 +- .../pherf/configuration/DataTypeMapping.java | 3 +- .../apache/phoenix/pherf/configuration/Query.java | 15 +- .../apache/phoenix/pherf/rules/RulesApplier.java | 215 +- .../pherf/rules/SequentialListDataGenerator.java | 1 + .../rules/SequentialVarcharDataGenerator.java | 1 + .../org/apache/phoenix/pherf/util/PhoenixUtil.java | 8 +- phoenix-server/pom.xml | 12 +- pom.xml | 71 +- 337 files changed, 27468 insertions(+), 4255 deletions(-) diff --cc phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java index 208961c0f1,879539473d..9879e76e0a --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java @@@ -78,8 -84,10 +84,9 @@@ import org.apache.phoenix.schema.Schema import org.apache.phoenix.schema.TableAlreadyExistsException; import org.apache.phoenix.schema.TableNotFoundException; import org.apache.phoenix.util.ByteUtil; + import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; import org.apache.phoenix.util.EnvironmentEdgeManager; import org.apache.phoenix.util.IndexUtil; -import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; @@@ -88,9 -98,23 +97,24 @@@ import org.junit.BeforeClass import org.junit.Test; import org.junit.experimental.categories.Category; - @Category({ParallelStatsDisabledTest.class}) ++ + @Category(NeedsOwnMiniClusterTest.class) public class CreateTableIT extends ParallelStatsDisabledIT { + @BeforeClass + public static synchronized void doSetup() throws Exception { + Map<String, String> props = Maps.newHashMapWithExpectedSize(1); + props.put(PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, Integer.toString(60*60)); // An hour + props.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(false)); + /** + * This test checks Table properties at ColumnFamilyDescriptor level, turing phoenix_table_ttl + * to false for them to test TTL and other props at HBase level. TTL being set at phoenix level + * is being tested in {@link TTLAsPhoenixTTLIT} + */ + props.put(QueryServices.PHOENIX_TABLE_TTL_ENABLED, Boolean.toString(false)); + setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); + } + @Test public void testStartKeyStopKey() throws SQLException { Properties props = new Properties(); diff --cc phoenix-core/src/it/java/org/apache/phoenix/end2end/DefaultPhoenixMultiViewListProviderIT.java index c90d30b534,1e69e3a048..6aabcbe358 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DefaultPhoenixMultiViewListProviderIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DefaultPhoenixMultiViewListProviderIT.java @@@ -34,7 -35,12 +35,16 @@@ import static org.apache.phoenix.mapred import static org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.MAPREDUCE_MULTI_INPUT_QUERY_BATCH_SIZE; import static org.junit.Assert.assertEquals; ++<<<<<<< HEAD +@Category({NeedsOwnMiniClusterTest.class}) ++======= + /** + * Disabling this test as this works on TTL being set on View which is removed and will be added in future. + * TODO:- To enable this test after re-enabling TTL for view for more info check :- PHOENIX-6978 + */ + @Ignore + @Category(NeedsOwnMiniClusterTest.class) ++>>>>>>> 1f88ca83f0efb576f01bdffb35becdc5ebccf58a public class DefaultPhoenixMultiViewListProviderIT extends ParallelStatsDisabledIT { private final String BASE_TABLE_DDL = "CREATE TABLE %s (TENANT_ID CHAR(10) NOT NULL, " + "ID CHAR(10) NOT NULL, NUM BIGINT CONSTRAINT " + diff --cc phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingAlterTableIT.java index 2e39e83602,055bee49cb..978a81d096 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingAlterTableIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingAlterTableIT.java @@@ -36,9 -43,23 +43,23 @@@ import org.junit.BeforeClass import org.junit.Test; import org.junit.experimental.categories.Category; - @Category({ParallelStatsDisabledTest.class, SupportsDistributedClusterTest.class}) -@Category(NeedsOwnMiniClusterTest.class) ++@Category({NeedsOwnMiniClusterTest.class, SupportsDistributedClusterTest.class}) public class FlappingAlterTableIT extends ParallelStatsDisabledIT { private String dataTableFullName; + + @BeforeClass + public static synchronized void doSetup() throws Exception { + Map<String, String> props = Maps.newHashMapWithExpectedSize(1); + props.put(PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, Integer.toString(60*60)); // An hour + props.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(false)); + /** + * This test checks Table properties at ColumnFamilyDescriptor level, turing phoenix_table_ttl + * to false for them to test TTL and other props at HBase level. TTL being set at phoenix level + * is being tested in {@link TTLAsPhoenixTTLIT} + */ + props.put(QueryServices.PHOENIX_TABLE_TTL_ENABLED, Boolean.toString(false)); + setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); + } @Before public void setupTableNames() throws Exception { diff --cc phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackExtendedIT.java index 09f13a81e8,1e2fa2752c..13eee5d23c --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackExtendedIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackExtendedIT.java @@@ -60,8 -63,11 +63,11 @@@ import static org.apache.phoenix.util.T import static org.apache.phoenix.util.TestUtil.assertRowHasExpectedValueAtSCN; import static org.apache.phoenix.util.TestUtil.assertTableHasTtl; import static org.apache.phoenix.util.TestUtil.assertTableHasVersions; + import static org.junit.Assert.assertEquals; + import static org.junit.Assert.assertFalse; + import static org.junit.Assert.assertTrue; -@Category(NeedsOwnMiniClusterTest.class) +@Category({NeedsOwnMiniClusterTest.class}) @RunWith(Parameterized.class) public class MaxLookbackExtendedIT extends BaseTest { private static final int MAX_LOOKBACK_AGE = 15; diff --cc phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByIT.java index 0236aac9c8,490dd45895..2ba3809beb --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByIT.java @@@ -17,9 -17,146 +17,145 @@@ */ package org.apache.phoenix.end2end; + import static org.apache.phoenix.util.TestUtil.assertResultSet; + import java.sql.Connection; + import java.sql.DriverManager; + import java.sql.ResultSet; + import org.junit.Test; import org.junit.experimental.categories.Category; - -@Category(ParallelStatsDisabledTest.class) +@Category({ParallelStatsDisabledTest.class, SupportsDistributedClusterTest.class}) public class OrderByIT extends BaseOrderByIT { + @Test + public void testPartialOrderForTupleProjectionPlanBug7352() throws Exception { + doTestPartialOrderForTupleProjectionPlanBug7352(false, false); + doTestPartialOrderForTupleProjectionPlanBug7352(false, true); + doTestPartialOrderForTupleProjectionPlanBug7352(true, false); + doTestPartialOrderForTupleProjectionPlanBug7352(true, true); + } + + private void doTestPartialOrderForTupleProjectionPlanBug7352( + boolean desc, boolean salted) throws Exception { + Connection conn = null; + try { + conn = DriverManager.getConnection(getUrl(), props); + String tableName = generateUniqueName(); + String sql = "create table " + tableName + "( "+ + " pk1 char(20) not null , " + + " pk2 char(20) not null, " + + " pk3 char(20) not null," + + " v1 varchar, " + + " v2 varchar, " + + " v3 varchar, " + + " CONSTRAINT TEST_PK PRIMARY KEY ( "+ + "pk1 "+(desc ? "desc" : "")+", "+ + "pk2 "+(desc ? "desc" : "")+", "+ + "pk3 "+(desc ? "desc" : "")+ + " )) "+(salted ? "SALT_BUCKETS =4" : "split on('b')"); + conn.createStatement().execute(sql); + + conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('a11','a12','a13','a14','a15','a16')"); + conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('a21','a22','a23','a24','a25','a26')"); + conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('a31','a32','a33','a34','a35','a36')"); + conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('b11','b12','b13','b14','b15','b16')"); + conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('b21','b22','b23','b24','b25', 'b26')"); + conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('b31','b32','b33','b34','b35', 'b36')"); + conn.commit(); + + sql = "select pk3,v1,v2 from (select v1,v2,pk3 from " + tableName + " t where pk1 > 'a10' order by t.v2,t.v1 limit 10) a order by v2"; + ResultSet rs = conn.prepareStatement(sql).executeQuery(); + assertResultSet(rs, new Object[][]{{"a13"},{"a23"},{"a33"},{"b13"},{"b23"},{"b33"}}); + + sql = "select pk3,v1,v2 from (select v1,v2,pk3 from " + tableName + + " t where pk1 > 'a10' order by t.v2 desc,t.v1 desc limit 10) a order by v2 desc"; + rs = conn.prepareStatement(sql).executeQuery(); + assertResultSet(rs, new Object[][]{{"b33"},{"b23"},{"b13"},{"a33"},{"a23"},{"a13"}}); + + sql = "select pk3,v1,v2 from (select v1,v2,pk3 from " + tableName + + " t where pk1 > 'a10' order by t.v2 desc,t.v1 desc, t.v3 desc limit 10) a order by v2 desc"; + rs = conn.prepareStatement(sql).executeQuery(); + assertResultSet(rs, new Object[][]{{"b33"},{"b23"},{"b13"},{"a33"},{"a23"},{"a13"}}); + + sql = "select pk3,v1,v2 from (select v1,v2,pk3 from " + tableName + + " t where pk1 > 'a10' order by t.v2 desc,t.v1 desc, t.v3 asc limit 10) a order by v2 desc"; + rs = conn.prepareStatement(sql).executeQuery(); + assertResultSet(rs, new Object[][]{{"b33"},{"b23"},{"b13"},{"a33"},{"a23"},{"a13"}}); + + sql = "select v2,cnt from (select count(pk3) cnt,v1,v2 from " + tableName + + " t where pk1 > 'a10' group by t.v1, t.v2, t.v3 limit 10) a order by v1"; + rs = conn.prepareStatement(sql).executeQuery(); + assertResultSet(rs, new Object[][]{{"a15"},{"a25"},{"a35"},{"b15"},{"b25"},{"b35"}}); + + sql = "select sub, pk2Cnt from (select substr(v2,0,3) sub, cast (count(pk3) as bigint) cnt, count(pk2) pk2Cnt from " + + tableName + + " t where pk1 > 'a10' group by t.v1 ,t.v2, t.v3 " + + " order by count(pk3) desc,t.v2 desc,t.v3 desc limit 10) a order by cnt desc ,sub desc"; + rs = conn.prepareStatement(sql).executeQuery(); + assertResultSet(rs, new Object[][]{{"b35"},{"b25"},{"b15"},{"a35"},{"a25"},{"a15"}}); + + sql = "select sub, pk2Cnt from (select substr(v2,0,3) sub,cast (count(pk3) as bigint) cnt, count(pk2) pk2Cnt from " + + tableName + + " t where pk1 > 'a10' group by t.v1 ,t.v2, t.v3 " + + " order by count(pk3) desc,t.v2 desc,t.v3 asc limit 10) a order by cnt desc ,sub desc"; + rs = conn.prepareStatement(sql).executeQuery(); + assertResultSet(rs, new Object[][]{{"b35"},{"b25"},{"b15"},{"a35"},{"a25"},{"a15"}}); + + sql = "select sub, pk2Cnt from (select substr(v2,0,3) sub,cast (count(pk3) as bigint) cnt, count(pk2) pk2Cnt from " + + tableName + + " t where pk1 > 'a10' group by t.v1 ,t.v2, t.v3 " + + " order by t.v2 desc, count(pk3) desc, t.v3 desc limit 10) a order by sub desc, cnt desc"; + rs = conn.prepareStatement(sql).executeQuery(); + assertResultSet(rs, new Object[][]{{"b35"},{"b25"},{"b15"},{"a35"},{"a25"},{"a15"}}); + + sql = "select sub, pk2Cnt from (select substr(v2,0,3) sub,cast (count(pk3) as bigint) cnt, count(pk2) pk2Cnt from " + + tableName + + " t where pk1 > 'a10' group by t.v1 ,t.v2, t.v3 " + + " order by t.v2 desc, count(pk3) desc, t.v3 asc limit 10) a order by sub desc, cnt desc"; + rs = conn.prepareStatement(sql).executeQuery(); + assertResultSet(rs, new Object[][]{{"b35"},{"b25"},{"b15"},{"a35"},{"a25"},{"a15"}}); + + sql = "select v1, pk3, v2 from (select v1,v2,pk3 from " + tableName + + " t where pk1 > 'a10' order by t.v2,t.v1, t.v3 limit 10) a order by v1"; + rs = conn.prepareStatement(sql).executeQuery(); + assertResultSet(rs, new Object[][]{{"a14"},{"a24"},{"a34"},{"b14"},{"b24"},{"b34"}}); + + sql = "select pk3,pk1,pk2 from (select pk1,pk2,pk3 from " + tableName + + " t where pk1 > 'a10' order by t.v2, t.v1, t.v3 limit 10) a order by pk3"; + rs = conn.prepareStatement(sql).executeQuery(); + assertResultSet(rs, new Object[][]{{"a13"},{"a23"},{"a33"},{"b13"},{"b23"},{"b33"}}); + + conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('b31','a12','a13','a14','a15','a16')"); + conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('b31','a22','a23','a24','a25','a26')"); + conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('b31','a32','a33','a34','a35','a36')"); + conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('b31','b12','b13','b14','b15','b16')"); + conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('b31','b22','b23','b24','b25', 'b26')"); + conn.commit(); + + sql = "select sub, v1 from (select substr(pk3,0,3) sub, pk2, v1 from " + + tableName + " t where pk1 = 'b31' order by pk2, pk3 limit 10) a order by pk2 desc ,sub desc"; + rs = conn.prepareStatement(sql).executeQuery(); + assertResultSet(rs, new Object[][]{{"b33"},{"b23"},{"b13"},{"a33"},{"a23"},{"a13"}}); + + sql = "select sub, v1 from (select substr(pk3,0,3) sub, pk2, v1 from " + + tableName + " t where pk1 = 'b31' order by pk2 desc, pk3 desc limit 10) a order by pk2 desc ,sub desc"; + rs = conn.prepareStatement(sql).executeQuery(); + assertResultSet(rs, new Object[][]{{"b33"},{"b23"},{"b13"},{"a33"},{"a23"},{"a13"}}); + + sql = "select sub, v1 from (select substr(pk2,0,3) sub, pk3, v1 from " + + tableName + " t where pk1 = 'b31' order by pk2, pk3 limit 10) a order by sub desc ,pk3 desc"; + rs = conn.prepareStatement(sql).executeQuery(); + assertResultSet(rs, new Object[][]{{"b32"},{"b22"},{"b12"},{"a32"},{"a22"},{"a12"}}); + + sql = "select sub, v1 from (select substr(pk2,0,3) sub, pk3, v1 from " + + tableName + " t where pk1 = 'b31' order by pk2 desc, pk3 desc limit 10) a order by sub desc, pk3 desc"; + rs = conn.prepareStatement(sql).executeQuery(); + assertResultSet(rs, new Object[][]{{"b32"},{"b22"},{"b12"},{"a32"},{"a22"},{"a12"}}); + } finally { + if(conn != null) { + conn.close(); + } + } + } + } diff --cc phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixTTLToolIT.java index 5f8105815e,d27cea12d6..0a2afc8dc3 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixTTLToolIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixTTLToolIT.java @@@ -49,7 -71,12 +71,13 @@@ import org.junit.BeforeClass import org.junit.Test; import org.junit.experimental.categories.Category; - @Category({NeedsOwnMiniClusterTest.class}) ++ + /** + * Disabling this test as this works on TTL being set on View which is removed and will be added in future. + * TODO:- To enable this test after re-enabling TTL for view for more info check :- PHOENIX-6978 + */ + @Ignore + @Category(NeedsOwnMiniClusterTest.class) public class PhoenixTTLToolIT extends ParallelStatsDisabledIT { @BeforeClass diff --cc phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewTTLIT.java index e3e5703b6d,db8ae88396..a59f474cd7 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewTTLIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewTTLIT.java @@@ -89,225 -91,43 +91,44 @@@ import static org.apache.phoenix.query. import static org.apache.phoenix.query.PhoenixTestBuilder.DDLDefaults.DEFAULT_SCHEMA_NAME; import static org.apache.phoenix.query.PhoenixTestBuilder.DDLDefaults.MAX_ROWS; import static org.apache.phoenix.query.PhoenixTestBuilder.DDLDefaults.TENANT_VIEW_COLUMNS; + import static org.apache.phoenix.query.PhoenixTestBuilder.DDLDefaults.TENANT_VIEW_INCLUDE_COLUMNS; + import static org.apache.phoenix.query.PhoenixTestBuilder.DDLDefaults.TENANT_VIEW_INDEX_COLUMNS; + import static org.apache.phoenix.query.PhoenixTestBuilder.DDLDefaults.TENANT_VIEW_PK_COLUMNS; + import static org.apache.phoenix.query.PhoenixTestBuilder.DDLDefaults.TENANT_VIEW_PK_TYPES; import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB; import static org.junit.Assert.assertEquals; - import static org.junit.Assert.assertNotNull; - import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; - @Category({NeedsOwnMiniClusterTest.class}) - public class ViewTTLIT extends ParallelStatsDisabledIT { - private static final Logger LOGGER = LoggerFactory.getLogger(ViewTTLIT.class); - private static final String ORG_ID_FMT = "00D0x000%s"; - static final String ID_FMT = "00A0y000%07d"; - static final String ZID_FMT = "00B0y000%07d"; - private static final String PHOENIX_TTL_HEADER_SQL = "SELECT PHOENIX_TTL FROM SYSTEM.CATALOG " - + "WHERE %s AND TABLE_SCHEM = '%s' AND TABLE_NAME = '%s' AND TABLE_TYPE = '%s'"; - - private static final String ALTER_PHOENIX_TTL_SQL - = "ALTER VIEW \"%s\".\"%s\" set PHOENIX_TTL=%s"; - - private static final String ALTER_SQL_WITH_NO_TTL - = "ALTER VIEW \"%s\".\"%s\" ADD IF NOT EXISTS %s CHAR(10)"; - private static final int DEFAULT_NUM_ROWS = 5; - - static final String COL1_FMT = "a%05d"; - static final String COL2_FMT = "b%05d"; - static final String COL3_FMT = "c%05d"; - static final String COL4_FMT = "d%05d"; - static final String COL5_FMT = "e%05d"; - static final String COL6_FMT = "f%05d"; - static final String COL7_FMT = "g%05d"; - static final String COL8_FMT = "h%05d"; - static final String COL9_FMT = "i%05d"; - - protected static void setUpTestDriver(ReadOnlyProps props) throws Exception { - setUpTestDriver(props, props); - } ++ + @Category(NeedsOwnMiniClusterTest.class) + public class ViewTTLIT extends BaseViewTTLIT { @BeforeClass - public static final void doSetup() throws Exception { - // Turn on the PHOENIX_TTL feature + public static void doSetup() throws Exception { + // Turn on the View TTL feature Map<String, String> DEFAULT_PROPERTIES = new HashMap<String, String>() {{ - put(QueryServices.PHOENIX_TTL_SERVER_SIDE_MASKING_ENABLED, String.valueOf(true)); + put(QueryServices.PHOENIX_TABLE_TTL_ENABLED, String.valueOf(true)); + put(QueryServices.LONG_VIEW_INDEX_ENABLED_ATTRIB, String.valueOf(false)); + // no max lookback + put(BaseScannerRegionObserverConstants.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, Integer.toString(0)); + put(QueryServices.PHOENIX_VIEW_TTL_ENABLED, Boolean.toString(true)); + put(QueryServices.PHOENIX_VIEW_TTL_TENANT_VIEWS_PER_SCAN_LIMIT, String.valueOf(1)); }}; - setUpTestDriver(new ReadOnlyProps(ReadOnlyProps.EMPTY_PROPS, DEFAULT_PROPERTIES.entrySet().iterator())); - } - - // Scans the HBase rows directly and asserts - private void assertUsingHBaseRows(byte[] hbaseTableName, - long minTimestamp, int expectedRows) throws IOException, SQLException { - - try (Table tbl = driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES) - .getTable(hbaseTableName)) { - - Scan allRows = new Scan(); - allRows.setTimeRange(minTimestamp, minTimestamp + Integer.MAX_VALUE); - ResultScanner scanner = tbl.getScanner(allRows); - int numMatchingRows = 0; - for (Result result = scanner.next(); result != null; result = scanner.next()) { - numMatchingRows++; - } - assertEquals(String.format("Expected rows do match for table = %s at timestamp %d", - Bytes.toString(hbaseTableName), minTimestamp), expectedRows, numMatchingRows); - } + setUpTestDriver(new ReadOnlyProps(ReadOnlyProps.EMPTY_PROPS, + DEFAULT_PROPERTIES.entrySet().iterator())); } - // Scans the HBase rows directly for the view ttl related header rows column and asserts - private void assertViewHeaderRowsHavePhoenixTTLRelatedCells(String schemaName, - long minTimestamp, boolean rawScan, int expectedRows) throws IOException, SQLException { - - FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL); - RowFilter schemaNameFilter = new RowFilter(CompareOperator.EQUAL, - new SubstringComparator(schemaName)); - QualifierFilter phoenixTTLQualifierFilter = new QualifierFilter( - CompareOperator.EQUAL, - new BinaryComparator(PhoenixDatabaseMetaData.PHOENIX_TTL_BYTES)); - filterList.addFilter(schemaNameFilter); - filterList.addFilter(phoenixTTLQualifierFilter); - try (Table tbl = driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES) - .getTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES)) { - - Scan allRows = new Scan(); - allRows.setRaw(rawScan); - allRows.setTimeRange(minTimestamp, HConstants.LATEST_TIMESTAMP); - allRows.setFilter(filterList); - ResultScanner scanner = tbl.getScanner(allRows); - int numMatchingRows = 0; - for (Result result = scanner.next(); result != null; result = scanner.next()) { - numMatchingRows += result.containsColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, - PhoenixDatabaseMetaData.PHOENIX_TTL_BYTES) ? 1 : 0; - } - assertEquals(String.format("Expected rows do not match for table = %s at timestamp %d", - Bytes.toString(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES), - minTimestamp), expectedRows, numMatchingRows); - } - - } - - private void assertSyscatHavePhoenixTTLRelatedColumns(String tenantId, String schemaName, - String tableName, String tableType, long ttlValueExpected) throws SQLException { - - try (Connection connection = DriverManager.getConnection(getUrl())) { - Statement stmt = connection.createStatement(); - String tenantClause = tenantId == null || tenantId.isEmpty() ? - "TENANT_ID IS NULL" : - String.format("TENANT_ID = '%s'", tenantId); - String sql = String - .format(PHOENIX_TTL_HEADER_SQL, tenantClause, schemaName, tableName, tableType); - stmt.execute(sql); - ResultSet rs = stmt.getResultSet(); - long actualTTLValueReturned = rs.next() ? rs.getLong(1) : 0; - - assertEquals(String.format("Expected rows do not match for schema = %s, table = %s", - schemaName, tableName), ttlValueExpected, actualTTLValueReturned); - } - } - - private String stripQuotes(String name) { - return name.replace("\"", ""); - } - - private SchemaBuilder createLevel2TenantViewWithGlobalLevelTTL( - TenantViewOptions tenantViewOptions, - TenantViewIndexOptions tenantViewIndexOptions, - boolean allowIndex) throws Exception { - // Define the test schema. - // 1. Table with columns => (ORG_ID, KP, COL1, COL2, COL3), PK => (ORG_ID, KP) - // 2. GlobalView with columns => (ID, COL4, COL5, COL6), PK => (ID) - // 3. Tenant with columns => (ZID, COL7, COL8, COL9), PK => (ZID) - final SchemaBuilder schemaBuilder = new SchemaBuilder(getUrl()); - - TableOptions tableOptions = TableOptions.withDefaults(); - tableOptions.setTableProps("COLUMN_ENCODED_BYTES=0,MULTI_TENANT=true"); - - GlobalViewOptions globalViewOptions = GlobalViewOptions.withDefaults(); - // Phoenix TTL is set to 300s => 300000 ms - globalViewOptions.setTableProps("PHOENIX_TTL=300"); - - SchemaBuilder.GlobalViewIndexOptions globalViewIndexOptions - = SchemaBuilder.GlobalViewIndexOptions.withDefaults(); - globalViewIndexOptions.setLocal(false); - - TenantViewOptions tenantViewWithOverrideOptions = TenantViewOptions.withDefaults(); - if (tenantViewOptions != null) { - tenantViewWithOverrideOptions = tenantViewOptions; - } - TenantViewIndexOptions tenantViewIndexOverrideOptions = TenantViewIndexOptions.withDefaults(); - if (tenantViewIndexOptions != null) { - tenantViewIndexOverrideOptions = tenantViewIndexOptions; - } - if (allowIndex) { - schemaBuilder.withTableOptions(tableOptions) - .withGlobalViewOptions(globalViewOptions) - .withGlobalViewIndexOptions(globalViewIndexOptions) - .withTenantViewOptions(tenantViewWithOverrideOptions) - .withTenantViewIndexOptions(tenantViewIndexOverrideOptions) - .buildWithNewTenant(); - } else { - schemaBuilder.withTableOptions(tableOptions) - .withGlobalViewOptions(globalViewOptions) - .withTenantViewOptions(tenantViewWithOverrideOptions) - .withTenantViewIndexOptions(tenantViewIndexOverrideOptions) - .buildWithNewTenant(); - } - return schemaBuilder; + @Before + public void beforeTest(){ + EnvironmentEdgeManager.reset(); + injectEdge = new ManualEnvironmentEdge(); + injectEdge.setValue(EnvironmentEdgeManager.currentTimeMillis()); } - private SchemaBuilder createLevel2TenantViewWithTenantLevelTTL( - TenantViewOptions tenantViewOptions, TenantViewIndexOptions tenantViewIndexOptions) - throws Exception { - // Define the test schema. - // 1. Table with columns => (ORG_ID, KP, COL1, COL2, COL3), PK => (ORG_ID, KP) - // 2. GlobalView with columns => (ID, COL4, COL5, COL6), PK => (ID) - // 3. Tenant with columns => (ZID, COL7, COL8, COL9), PK => (ZID) - final SchemaBuilder schemaBuilder = new SchemaBuilder(getUrl()); - - TableOptions tableOptions = TableOptions.withDefaults(); - tableOptions.setTableProps("COLUMN_ENCODED_BYTES=0,MULTI_TENANT=true"); - - GlobalViewOptions globalViewOptions = GlobalViewOptions.withDefaults(); - - TenantViewOptions tenantViewWithOverrideOptions = TenantViewOptions.withDefaults(); - // Phoenix TTL is set to 300s => 300000 ms - tenantViewWithOverrideOptions.setTableProps("PHOENIX_TTL=300"); - if (tenantViewOptions != null) { - tenantViewWithOverrideOptions = tenantViewOptions; - } - TenantViewIndexOptions tenantViewIndexOverrideOptions = TenantViewIndexOptions.withDefaults(); - if (tenantViewIndexOptions != null) { - tenantViewIndexOverrideOptions = tenantViewIndexOptions; - } - schemaBuilder.withTableOptions(tableOptions).withGlobalViewOptions(globalViewOptions) - .withTenantViewOptions(tenantViewWithOverrideOptions) - .withTenantViewIndexOptions(tenantViewIndexOverrideOptions).buildWithNewTenant(); - return schemaBuilder; - } - - private SchemaBuilder createLevel1TenantView(TenantViewOptions tenantViewOptions, - TenantViewIndexOptions tenantViewIndexOptions) throws Exception { - // Define the test schema. - // 1. Table with default columns => (ORG_ID, KP, COL1, COL2, COL3), PK => (ORG_ID, KP) - // 2. Tenant with default columns => (ZID, COL7, COL8, COL9), PK => (ZID) - final SchemaBuilder schemaBuilder = new SchemaBuilder(getUrl()); - - TableOptions tableOptions = TableOptions.withDefaults(); - tableOptions.setTableProps("COLUMN_ENCODED_BYTES=0,MULTI_TENANT=true"); - - TenantViewOptions tenantViewOverrideOptions = TenantViewOptions.withDefaults(); - if (tenantViewOptions != null) { - tenantViewOverrideOptions = tenantViewOptions; - } - TenantViewIndexOptions tenantViewIndexOverrideOptions = TenantViewIndexOptions.withDefaults(); - if (tenantViewIndexOptions != null) { - tenantViewIndexOverrideOptions = tenantViewIndexOptions; - } - - schemaBuilder.withTableOptions(tableOptions) - .withTenantViewOptions(tenantViewOverrideOptions) - .withTenantViewIndexOptions(tenantViewIndexOverrideOptions).buildNewView(); - return schemaBuilder; + @After + public synchronized void afterTest() throws Exception { + EnvironmentEdgeManager.reset(); } /** diff --cc phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewTTLNotEnabledIT.java index c7a81fa799,c4e5e677ab..9ace0d51cf --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewTTLNotEnabledIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewTTLNotEnabledIT.java @@@ -38,67 -49,80 +49,80 @@@ import java.util.Map import java.util.Properties; import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB; + import static org.junit.Assert.fail; -@Category(NeedsOwnMiniClusterTest.class) +@Category({NeedsOwnMiniClusterTest.class}) public class ViewTTLNotEnabledIT extends ParallelStatsDisabledIT { + @BeforeClass + public static synchronized void doSetup() throws Exception { + Map<String, String> props = Maps.newHashMapWithExpectedSize(1); + props.put(BaseScannerRegionObserverConstants.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, Integer.toString(60*60)); // An hour + props.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(false)); + props.put(QueryServices.PHOENIX_VIEW_TTL_ENABLED, Boolean.toString(false)); + setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); + } + @Test + public void testCreateViewWithTTLWithConfigFalse() throws Exception { + PhoenixTestBuilder.SchemaBuilder schemaBuilder = new PhoenixTestBuilder.SchemaBuilder(getUrl()); + PhoenixTestBuilder.SchemaBuilder.GlobalViewOptions globalViewOptions = PhoenixTestBuilder.SchemaBuilder. + GlobalViewOptions.withDefaults(); + globalViewOptions.setTableProps("TTL = 10000"); + try { + schemaBuilder.withTableOptions(PhoenixTestBuilder.SchemaBuilder.TableOptions.withDefaults()).withGlobalViewOptions( + globalViewOptions).build(); + fail(); + } catch (SQLException sqe) { + Assert.assertEquals(sqe.getErrorCode(), SQLExceptionCode.VIEW_TTL_NOT_ENABLED.getErrorCode()); + } + } + @Test - public void testPhoenixTTLNotEnabled() throws Exception { - - // PHOENIX TTL is set in seconds (for e.g 10 secs) - long phoenixTTL = 10; - PhoenixTestBuilder.SchemaBuilder.TableOptions - tableOptions = PhoenixTestBuilder.SchemaBuilder.TableOptions.withDefaults(); - tableOptions.getTableColumns().clear(); - tableOptions.getTableColumnTypes().clear(); - - PhoenixTestBuilder.SchemaBuilder.TenantViewOptions - tenantViewOptions = PhoenixTestBuilder.SchemaBuilder.TenantViewOptions.withDefaults(); - tenantViewOptions.setTableProps(String.format("PHOENIX_TTL=%d", phoenixTTL)); - - // Define the test schema. - final PhoenixTestBuilder.SchemaBuilder schemaBuilder = new PhoenixTestBuilder.SchemaBuilder(url); - schemaBuilder - .withTableOptions(tableOptions) - .withTenantViewOptions(tenantViewOptions) - .build(); - - String viewName = schemaBuilder.getEntityTenantViewName(); - - Properties props = new Properties(); - String tenantConnectUrl = - url + ';' + TENANT_ID_ATTRIB + '=' + schemaBuilder.getDataOptions().getTenantId(); - - // Test the coproc is not registered - org.apache.hadoop.hbase.client.Connection hconn = getUtility().getConnection(); - Admin admin = hconn.getAdmin(); - HTableDescriptor tableDescriptor = admin.getTableDescriptor( - TableName.valueOf(schemaBuilder.getEntityTableName())); - Assert.assertFalse("Coprocessor " + PhoenixTTLRegionObserver.class.getName() - + " should not have been added: ", - tableDescriptor.hasCoprocessor(PhoenixTTLRegionObserver.class.getName())); - - - // Test masking expired rows property are not set - try (Connection conn = DriverManager.getConnection(tenantConnectUrl, props); - final Statement statement = conn.createStatement()) { - conn.setAutoCommit(true); - - final String stmtString = String.format("select * from %s", viewName); - Preconditions.checkNotNull(stmtString); - final PhoenixStatement pstmt = statement.unwrap(PhoenixStatement.class); - final QueryPlan queryPlan = pstmt.optimizeQuery(stmtString); - - PhoenixResultSet - rs = pstmt.newResultSet(queryPlan.iterator(), queryPlan.getProjector(), queryPlan.getContext()); - Assert.assertFalse("Should not have any rows", rs.next()); - Assert.assertEquals("Should have at least one element", 1, queryPlan.getScans().size()); - Assert.assertEquals("PhoenixTTL should not be set", - 0, ScanUtil.getPhoenixTTL(queryPlan.getScans().get(0).get(0))); - Assert.assertFalse("Masking attribute should not be set", - ScanUtil.isMaskTTLExpiredRows(queryPlan.getScans().get(0).get(0))); - Assert.assertFalse("Delete Expired attribute should not set", - ScanUtil.isDeleteTTLExpiredRows(queryPlan.getScans().get(0).get(0))); + public void testAlterViewWithTTLWithConfigFalse() throws Exception { + PhoenixTestBuilder.SchemaBuilder schemaBuilder = new PhoenixTestBuilder.SchemaBuilder(getUrl()); + PhoenixTestBuilder.SchemaBuilder.GlobalViewOptions globalViewOptions = PhoenixTestBuilder.SchemaBuilder. + GlobalViewOptions.withDefaults(); + schemaBuilder.withTableOptions(PhoenixTestBuilder.SchemaBuilder.TableOptions.withDefaults()).withGlobalViewOptions( + globalViewOptions).build(); + + String dml = "ALTER VIEW " + schemaBuilder.getEntityGlobalViewName() + " SET TTL = 10000"; + try (Connection connection = DriverManager.getConnection(getUrl())){ + try { + connection.createStatement().execute(dml); + fail(); + } catch (SQLException sqe) { + Assert.assertEquals(sqe.getErrorCode(), SQLExceptionCode.VIEW_TTL_NOT_ENABLED.getErrorCode()); + } + + } + } + + @Test + public void testSettingTTLFromTableToViewWithConfigDisabled() throws Exception { + PhoenixTestBuilder.SchemaBuilder schemaBuilder = new PhoenixTestBuilder.SchemaBuilder(getUrl()); + PhoenixTestBuilder.SchemaBuilder.TableOptions tableOptions = PhoenixTestBuilder.SchemaBuilder.TableOptions.withDefaults(); + tableOptions.setTableProps("TTL = 10000"); + PhoenixTestBuilder.SchemaBuilder.GlobalViewOptions globalViewOptions = PhoenixTestBuilder.SchemaBuilder. + GlobalViewOptions.withDefaults(); + schemaBuilder.withTableOptions(tableOptions).withGlobalViewOptions( + globalViewOptions).build(); + + try (Connection connection = DriverManager.getConnection(getUrl())){ + + String dml = "ALTER TABLE " + schemaBuilder.getEntityTableName() + " SET TTL = NONE"; + connection.createStatement().execute(dml); + + //Clearing cache as metaDataCaching is not there for TTL usecase + connection.unwrap(PhoenixConnection.class).getQueryServices().clearCache(); + + try { + dml = "ALTER VIEW " + schemaBuilder.getEntityGlobalViewName() + " SET TTL = 10000"; + connection.createStatement().execute(dml); + fail(); + } catch (SQLException sqe) { + Assert.assertEquals(sqe.getErrorCode(), SQLExceptionCode.VIEW_TTL_NOT_ENABLED.getErrorCode()); + } + } } diff --cc phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java index 95a7c8c922,1405ad6355..3b448fcaf6 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java @@@ -17,7 -17,7 +17,8 @@@ */ package org.apache.phoenix.query; +import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_CLIENT_PORT; + import static org.apache.hadoop.hbase.coprocessor.CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY; import static org.apache.phoenix.hbase.index.write.ParallelWriterIndexCommitter.NUM_CONCURRENT_INDEX_WRITER_THREADS_CONF_KEY; import static org.apache.phoenix.query.QueryConstants.MILLIS_IN_DAY; import static org.apache.phoenix.query.QueryServices.DROP_METADATA_ATTRIB; @@@ -138,8 -140,8 +139,9 @@@ import org.apache.hadoop.hbase.regionse import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.RSRpcServices; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.PhoenixTestTableName; import org.apache.phoenix.SystemExitRule; + import org.apache.phoenix.compat.hbase.CompatUtil; import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; import org.apache.phoenix.end2end.ParallelStatsDisabledIT; import org.apache.phoenix.end2end.ParallelStatsDisabledTest;