Separate index build from index access This change separates index build from index access. All indexes now have a single dataflow helper which uses the index path to locate the resource on the nc to read the resource from memory or disk.
Existing resource metadata and dataflow helpers were combined into resource builders eliminating lots of duplicated code. Change-Id: Ie4ea3aaa63dff8d246fa43ca7c7359729bc8cf47 Reviewed-on: https://asterix-gerrit.ics.uci.edu/1728 Integration-Tests: Ian Maxon <ima...@apache.org> Tested-by: Ian Maxon <ima...@apache.org> Reviewed-by: Yingyi Bu <buyin...@gmail.com> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/735532e4 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/735532e4 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/735532e4 Branch: refs/heads/master Commit: 735532e43aceb1fd806676ec4f537911ae29de4d Parents: 4671f71 Author: Abdullah Alamoudi <bamou...@gmail.com> Authored: Thu May 11 13:58:53 2017 -0700 Committer: abdullah alamoudi <bamou...@gmail.com> Committed: Thu May 11 16:42:33 2017 -0700 ---------------------------------------------------------------------- asterixdb/asterix-algebra/pom.xml | 4 - .../physical/InvertedIndexPOperator.java | 112 +--- .../rules/RemoveUnusedOneToOneEquiJoinRule.java | 3 +- .../optimizer/rules/UnnestToDataScanRule.java | 17 +- .../am/AbstractIntroduceAccessMethodRule.java | 14 +- .../optimizer/rules/am/AccessMethodUtils.java | 72 +-- .../am/IntroduceLSMComponentFilterRule.java | 28 +- .../rules/am/OptimizableOperatorSubTree.java | 22 +- .../LangExpressionToPlanTranslator.java | 4 +- .../AppRuntimeContextProviderForRecovery.java | 4 +- .../api/http/server/ConnectorApiServlet.java | 3 +- .../asterix/app/nc/NCAppRuntimeContext.java | 4 +- .../apache/asterix/app/nc/RecoveryManager.java | 13 +- .../asterix/app/nc/TransactionSubsystem.java | 2 +- .../asterix/app/translator/QueryTranslator.java | 22 +- .../asterix/file/StorageComponentProvider.java | 6 - .../hyracks/bootstrap/NCApplication.java | 2 +- .../apache/asterix/utils/FeedOperations.java | 9 +- .../app/bootstrap/TestNodeController.java | 188 ++---- .../asterix/test/dataflow/LogMarkerTest.java | 5 +- .../asterix/test/logging/CheckpointingTest.java | 5 +- .../common/api/IDatasetLifecycleManager.java | 12 +- .../common/api/INcApplicationContext.java | 4 +- .../AsterixVirtualBufferCacheProvider.java | 28 +- .../common/context/BaseOperationTracker.java | 4 +- .../context/CorrelatedPrefixMergePolicy.java | 17 +- .../CorrelatedPrefixMergePolicyFactory.java | 38 +- .../common/context/DatasetLifecycleManager.java | 51 +- .../asterix/common/context/DatasetResource.java | 2 +- .../context/IStorageComponentProvider.java | 7 - .../context/PrimaryIndexOperationTracker.java | 8 +- .../common/dataflow/DatasetLocalResource.java | 65 ++ .../common/dataflow/ICcApplicationContext.java | 33 +- .../LSMInsertDeleteOperatorNodePushable.java | 20 +- ...rtedIndexInsertDeleteOperatorDescriptor.java | 73 --- .../LSMTreeInsertDeleteOperatorDescriptor.java | 52 +- .../asterix/common/exceptions/ErrorCode.java | 2 + .../asterix/common/metadata/IDataset.java | 12 + .../IAppRuntimeContextProvider.java | 2 +- .../common/transactions/IResourceFactory.java | 32 - .../asterix/common/transactions/Resource.java | 88 --- .../common/transactions/ResourceFactory.java | 50 -- .../asterix/common/utils/StoragePathUtil.java | 6 + .../main/resources/asx_errormsg/en.properties | 2 + .../indexing/ExternalFileIndexAccessor.java | 32 +- .../indexing/FileIndexTupleTranslator.java | 21 +- .../indexing/FilesIndexDescription.java | 61 +- .../external/indexing/IndexingConstants.java | 34 +- ...xternalDatasetIndexesOperatorDescriptor.java | 28 +- .../ExternalBTreeSearchOperatorDescriptor.java | 37 +- ...ExternalBTreeSearchOperatorNodePushable.java | 32 +- ...alDatasetIndexesAbortOperatorDescriptor.java | 21 +- ...lDatasetIndexesCommitOperatorDescriptor.java | 27 +- ...DatasetIndexesRecoverOperatorDescriptor.java | 22 +- ...ernalFilesIndexCreateOperatorDescriptor.java | 111 ++++ ...ilesIndexModificationOperatorDescriptor.java | 124 ++++ .../ExternalFilesIndexOperatorDescriptor.java | 162 ----- ...ExternalIndexBulkLoadOperatorDescriptor.java | 51 ++ ...ternalIndexBulkLoadOperatorNodePushable.java | 46 ++ ...ternalIndexBulkModifyOperatorDescriptor.java | 39 +- ...rnalIndexBulkModifyOperatorNodePushable.java | 18 +- .../ExternalLookupOperatorDescriptor.java | 36 +- .../ExternalRTreeSearchOperatorDescriptor.java | 31 +- ...ExternalRTreeSearchOperatorNodePushable.java | 32 +- .../ExternalScanOperatorDescriptor.java | 2 +- .../FeedCollectOperatorDescriptor.java | 4 +- .../operators/FeedIntakeOperatorDescriptor.java | 4 +- .../operators/FeedMetaComputeNodePushable.java | 4 +- .../operators/FeedMetaOperatorDescriptor.java | 11 +- .../operators/FeedMetaStoreNodePushable.java | 16 +- .../operators/IndexInfoOperatorDescriptor.java | 130 ---- .../apache/asterix/external/util/FeedUtils.java | 4 +- .../apache/asterix/metadata/MetadataNode.java | 10 +- .../IIndexDataflowHelperFactoryProvider.java | 65 -- .../metadata/api/IResourceFactoryProvider.java | 65 ++ .../metadata/bootstrap/MetadataBootstrap.java | 84 +-- .../BTreeDataflowHelperFactoryProvider.java | 93 --- .../declared/BTreeResourceFactoryProvider.java | 194 ++++++ .../metadata/declared/LoadableDataSource.java | 3 +- .../metadata/declared/MetadataProvider.java | 594 +++---------------- .../asterix/metadata/entities/Dataset.java | 134 ++++- .../asterix/metadata/utils/DatasetUtil.java | 190 ++---- .../utils/ExternalIndexingOperations.java | 254 +++----- .../asterix/metadata/utils/IndexUtil.java | 25 +- ...ertedIndexDataflowHelperFactoryProvider.java | 117 ---- .../InvertedIndexResourceFactoryProvider.java | 248 ++++++++ .../RTreeDataflowHelperFactoryProvider.java | 172 ------ .../utils/RTreeResourceFactoryProvider.java | 258 ++++++++ .../utils/SecondaryBTreeOperationsHelper.java | 76 +-- .../utils/SecondaryIndexOperationsHelper.java | 70 +-- .../SecondaryInvertedIndexOperationsHelper.java | 122 +--- .../utils/SecondaryRTreeOperationsHelper.java | 113 +--- .../SecondaryTreeIndexOperationsHelper.java | 58 +- .../utils/SplitsAndConstraintsUtil.java | 83 +-- .../BinaryComparatorFactoryProvider.java | 1 + .../storage/ReplicaResourcesManager.java | 4 +- .../runtime/base/AsterixTupleFilterFactory.java | 3 +- ...SMInvertedIndexUpsertOperatorDescriptor.java | 68 --- .../LSMPrimaryUpsertOperatorDescriptor.java | 73 +++ .../LSMPrimaryUpsertOperatorNodePushable.java | 34 +- .../LSMSecondaryUpsertOperatorDescriptor.java | 54 ++ .../LSMSecondaryUpsertOperatorNodePushable.java | 22 +- .../LSMTreeUpsertOperatorDescriptor.java | 88 --- .../runtime/utils/CcApplicationContext.java | 6 - .../utils/RuntimeComponentsProvider.java | 40 +- asterixdb/asterix-transactions/pom.xml | 16 - ...tractIndexModificationOperationCallback.java | 2 +- .../LockThenSearchOperationCallback.java | 2 +- .../LockThenSearchOperationCallbackFactory.java | 3 +- ...maryIndexInstantSearchOperationCallback.java | 2 +- ...exInstantSearchOperationCallbackFactory.java | 2 +- ...dexModificationOperationCallbackFactory.java | 14 +- .../PrimaryIndexOperationTrackerFactory.java | 10 +- .../PrimaryIndexSearchOperationCallback.java | 2 +- ...maryIndexSearchOperationCallbackFactory.java | 2 +- ...dexModificationOperationCallbackFactory.java | 12 +- .../SecondaryIndexSearchOperationCallback.java | 2 +- ...daryIndexSearchOperationCallbackFactory.java | 2 +- ...dexModificationOperationCallbackFactory.java | 12 +- ...dexModificationOperationCallbackFactory.java | 15 +- .../UpsertOperationCallbackFactory.java | 12 +- .../resource/DatasetLocalResourceFactory.java | 45 ++ .../ExternalBTreeLocalResourceMetadata.java | 66 --- ...ternalBTreeLocalResourceMetadataFactory.java | 52 -- ...rnalBTreeWithBuddyLocalResourceMetadata.java | 81 --- ...eeWithBuddyLocalResourceMetadataFactory.java | 61 -- .../ExternalRTreeLocalResourceMetadata.java | 74 --- ...ternalRTreeLocalResourceMetadataFactory.java | 59 -- .../resource/LSMBTreeLocalResourceMetadata.java | 94 --- .../LSMBTreeLocalResourceMetadataFactory.java | 68 --- .../LSMInvertedIndexLocalResourceMetadata.java | 112 ---- ...vertedIndexLocalResourceMetadataFactory.java | 82 --- .../resource/LSMRTreeLocalResourceMetadata.java | 100 ---- .../LSMRTreeLocalResourceMetadataFactory.java | 81 --- .../PersistentLocalResourceFactory.java | 41 -- .../PersistentLocalResourceFactoryProvider.java | 40 -- .../PersistentLocalResourceRepository.java | 100 ++-- ...ersistentLocalResourceRepositoryFactory.java | 2 +- .../recovery/AbstractCheckpointManager.java | 2 + .../service/transaction/TransactionContext.java | 2 +- .../locking/TestRuntimeContextProvider.java | 2 +- .../algebra/metadata/IMetadataProvider.java | 4 +- .../data/IBinaryComparatorFactoryProvider.java | 17 +- .../meta/AlgebricksMetaOperatorDescriptor.java | 6 +- .../operators/std/SplitOperatorDescriptor.java | 2 +- .../api/context/IHyracksCommonContext.java | 2 +- .../api/context/IHyracksTaskContext.java | 18 +- .../hyracks/api/dataflow/value/IComparator.java | 23 - .../api/dataflow/value/IComparatorFactory.java | 25 - .../hyracks/api/dataflow/value/ITypeTraits.java | 13 +- .../hyracks/client/dataset/HyracksDataset.java | 2 +- .../org/apache/hyracks/control/nc/Task.java | 2 +- .../nc/dataset/DatasetPartitionWriter.java | 2 +- .../partitions/MaterializedPartitionWriter.java | 8 +- .../MaterializingPipelinedPartition.java | 6 +- .../std/base/AbstractOperatorDescriptor.java | 6 +- .../AbstractReplicateOperatorDescriptor.java | 2 +- ...tDeserializedFileScanOperatorDescriptor.java | 6 +- .../AbstractFileWriteOperatorDescriptor.java | 2 +- .../std/file/FileRemoveOperatorDescriptor.java | 2 +- .../std/file/FileScanOperatorDescriptor.java | 4 +- .../file/FrameFileWriterOperatorDescriptor.java | 2 +- .../file/PlainFileWriterOperatorDescriptor.java | 2 +- .../DeserializedPreclusteredGroupOperator.java | 137 ----- .../ExternalGroupBuildOperatorNodePushable.java | 2 +- .../ExternalGroupOperatorDescriptor.java | 4 +- .../ExternalGroupWriteOperatorNodePushable.java | 2 +- .../PreclusteredGroupOperatorDescriptor.java | 4 +- .../sort/ExternalSortGroupByRunGenerator.java | 2 +- .../sort/ExternalSortGroupByRunMerger.java | 2 +- .../intersect/IntersectOperatorDescriptor.java | 2 +- .../join/HybridHashJoinOperatorDescriptor.java | 6 +- .../InMemoryHashJoinOperatorDescriptor.java | 4 +- .../dataflow/std/join/NestedLoopJoin.java | 2 +- .../join/NestedLoopJoinOperatorDescriptor.java | 2 +- .../std/join/OptimizedHybridHashJoin.java | 2 +- ...timizedHybridHashJoinOperatorDescriptor.java | 2 +- .../DeserializedMapperOperatorDescriptor.java | 2 +- .../ConstantTupleSourceOperatorDescriptor.java | 2 +- .../std/misc/IdentityOperatorDescriptor.java | 2 +- .../std/misc/LimitOperatorDescriptor.java | 4 +- .../std/misc/MaterializerTaskState.java | 2 +- .../misc/MaterializingOperatorDescriptor.java | 2 +- .../std/misc/SplitVectorOperatorDescriptor.java | 2 +- .../sort/AbstractSorterOperatorDescriptor.java | 2 +- .../sort/ExternalSortOperatorDescriptor.java | 4 +- .../std/sort/ExternalSortRunGenerator.java | 2 +- .../std/sort/ExternalSortRunMerger.java | 2 +- .../dataflow/std/sort/HeapSortRunGenerator.java | 2 +- .../std/sort/HybridTopKSortRunGenerator.java | 2 +- .../sort/InMemorySortOperatorDescriptor.java | 4 +- .../std/sort/TopKSorterOperatorDescriptor.java | 4 +- .../std/union/UnionAllOperatorDescriptor.java | 2 +- .../btree/client/InsertPipelineExample.java | 58 +- .../client/PrimaryIndexBulkLoadExample.java | 46 +- .../btree/client/PrimaryIndexSearchExample.java | 17 +- .../client/SecondaryIndexBulkLoadExample.java | 33 +- .../client/SecondaryIndexSearchExample.java | 33 +- .../btree/helper/BTreeHelperStorageManager.java | 19 +- .../btree/helper/DataGenOperatorDescriptor.java | 4 +- .../helper/IndexLifecycleManagerProvider.java | 33 -- .../examples/btree/helper/RuntimeContext.java | 18 +- .../am/btree/AbstractBTreeOperatorTest.java | 141 ++--- .../tests/am/btree/BTreeOperatorTestHelper.java | 32 - .../BTreePrimaryIndexScanOperatorTest.java | 41 +- .../BTreePrimaryIndexSearchOperatorTest.java | 42 +- .../BTreePrimaryIndexStatsOperatorTest.java | 29 +- .../BTreeSecondaryIndexInsertOperatorTest.java | 43 +- .../BTreeSecondaryIndexSearchOperatorTest.java | 43 +- .../BTreeSecondaryIndexUpsertOperatorTest.java | 43 +- .../am/common/ITreeIndexOperatorTestHelper.java | 5 +- .../am/common/LSMTreeOperatorTestHelper.java | 22 +- .../lsm/btree/LSMBTreeOperatorTestHelper.java | 41 +- .../LSMBTreePrimaryIndexScanOperatorTest.java | 21 +- .../LSMBTreePrimaryIndexSearchOperatorTest.java | 68 +-- ...SMBTreeSecondaryIndexInsertOperatorTest.java | 21 +- ...SMBTreeSecondaryIndexSearchOperatorTest.java | 62 +- .../lsm/rtree/LSMRTreeOperatorTestHelper.java | 36 +- ...SMRTreeSecondaryIndexInsertOperatorTest.java | 15 +- ...SMRTreeSecondaryIndexSearchOperatorTest.java | 15 +- ...eWithAntiMatterTuplesOperatorTestHelper.java | 36 +- ...rTuplesSecondaryIndexInsertOperatorTest.java | 15 +- ...rTuplesSecondaryIndexSearchOperatorTest.java | 39 +- .../am/rtree/AbstractRTreeOperatorTest.java | 164 +++-- .../tests/am/rtree/RTreeOperatorTestHelper.java | 18 +- .../RTreeSecondaryIndexInsertOperatorTest.java | 39 +- .../RTreeSecondaryIndexScanOperatorTest.java | 42 +- .../RTreeSecondaryIndexSearchOperatorTest.java | 39 +- .../RTreeSecondaryIndexStatsOperatorTest.java | 31 +- .../tests/integration/JobFailureTest.java | 2 +- .../dataflow/HDFSReadOperatorDescriptor.java | 2 +- .../dataflow/HDFSReadOperatorDescriptor.java | 2 +- .../hyracks-storage-am-bloomfilter/pom.xml | 5 - .../am/bloomfilter/impls/BloomFilter.java | 2 +- .../hyracks/hyracks-storage-am-btree/pom.xml | 9 +- .../storage/am/btree/api/IBTreeLeafFrame.java | 2 +- .../am/btree/api/IPrefixSlotManager.java | 2 +- .../compressors/FieldPrefixCompressor.java | 2 +- .../am/btree/dataflow/BTreeDataflowHelper.java | 51 -- .../dataflow/BTreeDataflowHelperFactory.java | 43 -- .../am/btree/dataflow/BTreeResource.java | 67 +++ .../am/btree/dataflow/BTreeResourceFactory.java | 51 ++ .../dataflow/BTreeSearchOperatorDescriptor.java | 74 +-- .../BTreeSearchOperatorNodePushable.java | 23 +- .../BTreeUpdateSearchOperatorDescriptor.java | 30 +- .../BTreeUpdateSearchOperatorNodePushable.java | 19 +- .../frames/BTreeFieldPrefixNSMLeafFrame.java | 2 +- .../am/btree/frames/BTreeNSMInteriorFrame.java | 2 +- .../am/btree/frames/BTreeNSMLeafFrame.java | 2 +- .../am/btree/frames/OrderedSlotManager.java | 2 +- .../hyracks/storage/am/btree/impls/BTree.java | 19 +- .../btree/impls/BTreeCountingSearchCursor.java | 6 +- .../am/btree/impls/BTreeCursorInitialState.java | 8 +- .../storage/am/btree/impls/BTreeOpContext.java | 8 +- .../am/btree/impls/BTreeRangeSearchCursor.java | 10 +- .../am/btree/impls/FieldPrefixSlotManager.java | 2 +- .../storage/am/btree/impls/RangePredicate.java | 2 +- .../storage/am/btree/util/BTreeUtils.java | 2 +- .../storage/am/btree/test/FramewriterTest.java | 47 +- .../am/common/api/ICursorInitialState.java | 37 -- .../hyracks/storage/am/common/api/IIndex.java | 134 ----- .../storage/am/common/api/IIndexAccessor.java | 105 ---- .../storage/am/common/api/IIndexBuilder.java | 35 ++ .../am/common/api/IIndexBuilderFactory.java | 29 + .../storage/am/common/api/IIndexBulkLoader.java | 51 -- .../storage/am/common/api/IIndexCursor.java | 81 --- .../am/common/api/IIndexDataflowHelper.java | 39 +- .../api/IIndexLifecycleManagerProvider.java | 31 - .../api/IModificationOperationCallback.java | 52 -- .../IModificationOperationCallbackFactory.java | 3 +- .../common/api/IResourceLifecycleManager.java | 83 --- .../am/common/api/ISearchOperationCallback.java | 79 --- .../api/ISearchOperationCallbackFactory.java | 1 + .../storage/am/common/api/ISearchPredicate.java | 40 -- .../storage/am/common/api/ISlotManager.java | 2 +- .../storage/am/common/api/ITreeIndex.java | 1 + .../am/common/api/ITreeIndexAccessor.java | 1 + .../storage/am/common/api/ITreeIndexCursor.java | 1 + .../storage/am/common/api/ITreeIndexFrame.java | 7 +- .../common/api/ITreeIndexFrameCompressor.java | 2 +- .../am/common/api/ITupleFilterFactory.java | 3 +- .../storage/am/common/build/IndexBuilder.java | 97 +++ .../am/common/build/IndexBuilderFactory.java | 53 ++ .../AbstractIndexOperatorDescriptor.java | 144 ----- .../AbstractTreeIndexOperatorDescriptor.java | 85 --- .../dataflow/IIndexDataflowHelperFactory.java | 4 +- .../dataflow/IIndexOperatorDescriptor.java | 60 -- .../IndexBulkLoadOperatorNodePushable.java | 24 +- .../dataflow/IndexCreateOperatorDescriptor.java | 46 ++ .../IndexCreateOperatorNodePushable.java | 14 +- .../am/common/dataflow/IndexDataflowHelper.java | 125 ++-- .../dataflow/IndexDataflowHelperFactory.java | 46 ++ .../dataflow/IndexDropOperatorDescriptor.java | 31 +- .../dataflow/IndexDropOperatorNodePushable.java | 4 +- ...xInsertUpdateDeleteOperatorNodePushable.java | 34 +- .../common/dataflow/IndexLifecycleManager.java | 4 +- .../IndexSearchOperatorNodePushable.java | 54 +- .../TreeIndexBulkLoadOperatorDescriptor.java | 43 +- .../TreeIndexCreateOperatorDescriptor.java | 59 -- .../dataflow/TreeIndexDataflowHelper.java | 42 -- ...reeIndexDiskOrderScanOperatorDescriptor.java | 30 +- ...eIndexDiskOrderScanOperatorNodePushable.java | 34 +- ...dexInsertUpdateDeleteOperatorDescriptor.java | 41 +- .../TreeIndexStatsOperatorDescriptor.java | 33 +- .../TreeIndexStatsOperatorNodePushable.java | 36 +- .../common/impls/AbstractSearchPredicate.java | 2 +- .../am/common/impls/AbstractTreeIndex.java | 4 +- .../am/common/impls/NoOpOperationCallback.java | 4 +- .../impls/NoOpOperationCallbackFactory.java | 6 +- .../impls/TreeIndexDiskOrderScanCursor.java | 4 +- .../am/common/ophelpers/MultiComparator.java | 133 ----- .../am/common/ophelpers/SingleComparator.java | 42 -- .../am/common/util/IndexFileNameUtil.java | 6 +- .../dataflow/ExternalBTreeDataflowHelper.java | 82 --- .../ExternalBTreeDataflowHelperFactory.java | 56 -- .../dataflow/ExternalBTreeLocalResource.java | 66 +++ .../ExternalBTreeLocalResourceFactory.java | 59 ++ .../ExternalBTreeWithBuddyDataflowHelper.java | 92 --- ...rnalBTreeWithBuddyDataflowHelperFactory.java | 57 -- .../ExternalBTreeWithBuddyLocalResource.java | 69 +++ ...ernalBTreeWithBuddyLocalResourceFactory.java | 59 ++ .../btree/dataflow/LSMBTreeDataflowHelper.java | 84 --- .../dataflow/LSMBTreeDataflowHelperFactory.java | 65 -- .../btree/dataflow/LSMBTreeLocalResource.java | 80 +++ .../dataflow/LSMBTreeLocalResourceFactory.java | 68 +++ .../am/lsm/btree/impls/ExternalBTree.java | 29 +- .../lsm/btree/impls/ExternalBTreeOpContext.java | 8 +- .../lsm/btree/impls/ExternalBTreeWithBuddy.java | 26 +- .../impls/ExternalBTreeWithBuddyOpContext.java | 8 +- .../storage/am/lsm/btree/impls/LSMBTree.java | 14 +- .../btree/impls/LSMBTreeCursorInitialState.java | 8 +- .../am/lsm/btree/impls/LSMBTreeOpContext.java | 8 +- .../btree/impls/LSMBTreePointSearchCursor.java | 6 +- .../btree/impls/LSMBTreeRangeSearchCursor.java | 8 +- .../lsm/btree/impls/LSMBTreeSearchCursor.java | 4 +- .../impls/LSMBTreeWithBuddyAbstractCursor.java | 6 +- .../LSMBTreeWithBuddyCursorInitialState.java | 6 +- .../impls/LSMBTreeWithBuddySearchCursor.java | 4 +- .../impls/LSMBTreeWithBuddySortedCursor.java | 4 +- .../btree/impls/LSMBuddyBTreeMergeCursor.java | 8 +- .../am/lsm/btree/utils/LSMBTreeUtil.java | 29 +- .../am/lsm/common/api/ILSMComponentFilter.java | 2 +- .../storage/am/lsm/common/api/ILSMHarness.java | 4 +- .../api/ILSMIOOperationCallbackProvider.java | 24 - .../api/ILSMIOOperationSchedulerProvider.java | 4 +- .../storage/am/lsm/common/api/ILSMIndex.java | 10 +- .../am/lsm/common/api/ILSMIndexAccessor.java | 2 +- .../common/api/ILSMIndexOperationContext.java | 6 +- .../lsm/common/api/ILSMMergePolicyFactory.java | 11 +- .../am/lsm/common/api/ILSMOperationTracker.java | 6 +- .../storage/am/lsm/common/api/ITwoPCIndex.java | 9 +- .../common/api/IVirtualBufferCacheProvider.java | 6 +- .../AbstractLSMIndexDataflowHelper.java | 85 --- .../AbstractLSMIndexDataflowHelperFactory.java | 66 --- .../LSMIndexCompactOperatorNodePushable.java | 16 +- ...xInsertUpdateDeleteOperatorNodePushable.java | 16 +- .../LSMTreeIndexCompactOperatorDescriptor.java | 29 +- ...dexInsertUpdateDeleteOperatorDescriptor.java | 42 +- .../am/lsm/common/dataflow/LsmResource.java | 101 ++++ .../lsm/common/dataflow/LsmResourceFactory.java | 74 +++ .../am/lsm/common/impls/AbstractLSMIndex.java | 9 +- .../lsm/common/impls/ConstantMergePolicy.java | 2 +- .../impls/ConstantMergePolicyFactory.java | 19 +- .../lsm/common/impls/ExternalIndexHarness.java | 4 +- .../am/lsm/common/impls/IndexFactory.java | 2 +- .../am/lsm/common/impls/LSMComponentFilter.java | 2 +- .../common/impls/LSMComponentFilterManager.java | 2 +- .../storage/am/lsm/common/impls/LSMHarness.java | 4 +- .../lsm/common/impls/LSMIndexSearchCursor.java | 10 +- .../lsm/common/impls/LSMTreeIndexAccessor.java | 4 +- .../lsm/common/impls/NoMergePolicyFactory.java | 16 +- .../common/impls/NoOpIOOperationCallback.java | 66 --- .../impls/NoOpIOOperationCallbackFactory.java | 66 +++ .../impls/NoOpOperationTrackerFactory.java | 4 +- .../common/impls/PrefixMergePolicyFactory.java | 20 +- .../impls/SynchronousSchedulerProvider.java | 4 +- .../lsm/common/impls/ThreadCountingTracker.java | 6 +- .../lsm/invertedindex/api/IInvertedIndex.java | 2 +- .../api/IInvertedIndexAccessor.java | 6 +- .../api/IInvertedIndexOperatorDescriptor.java | 38 -- .../invertedindex/api/IInvertedListCursor.java | 2 +- ...tractLSMInvertedIndexOperatorDescriptor.java | 97 --- .../BinaryTokenizerOperatorDescriptor.java | 4 +- ...InvertedIndexBulkLoadOperatorDescriptor.java | 73 --- .../LSMInvertedIndexCompactOperator.java | 62 -- ...SMInvertedIndexCreateOperatorDescriptor.java | 64 -- .../LSMInvertedIndexDataflowHelper.java | 95 --- .../LSMInvertedIndexDataflowHelperFactory.java | 70 --- .../LSMInvertedIndexDropOperatorDescriptor.java | 58 -- ...InvertedIndexInsertUpdateDeleteOperator.java | 76 --- .../dataflow/LSMInvertedIndexLocalResource.java | 110 ++++ .../LSMInvertedIndexLocalResourceFactory.java | 81 +++ ...SMInvertedIndexSearchOperatorDescriptor.java | 76 +-- ...InvertedIndexSearchOperatorNodePushable.java | 39 +- ...rtitionedLSMInvertedIndexDataflowHelper.java | 94 --- ...edLSMInvertedIndexDataflowHelperFactory.java | 69 --- .../invertedindex/impls/LSMInvertedIndex.java | 16 +- .../impls/LSMInvertedIndexAccessor.java | 4 +- ...nvertedIndexDeletedKeysBTreeMergeCursor.java | 10 +- .../impls/LSMInvertedIndexMergeOperation.java | 2 +- .../impls/LSMInvertedIndexOpContext.java | 10 +- .../LSMInvertedIndexRangeSearchCursor.java | 10 +- ...ertedIndexRangeSearchCursorInitialState.java | 10 +- .../impls/LSMInvertedIndexSearchCursor.java | 12 +- ...SMInvertedIndexSearchCursorInitialState.java | 8 +- .../inmemory/InMemoryInvertedIndex.java | 13 +- .../inmemory/InMemoryInvertedIndexAccessor.java | 4 +- .../InMemoryInvertedIndexOpContext.java | 2 +- .../inmemory/InMemoryInvertedListCursor.java | 4 +- .../PartitionedInMemoryInvertedIndex.java | 6 +- .../FixedSizeElementInvertedListCursor.java | 2 +- .../ondisk/OnDiskInvertedIndex.java | 21 +- .../ondisk/OnDiskInvertedIndexOpContext.java | 6 +- .../OnDiskInvertedIndexRangeSearchCursor.java | 8 +- .../ondisk/OnDiskInvertedIndexSearchCursor.java | 6 +- .../ondisk/PartitionedOnDiskInvertedIndex.java | 6 +- .../search/AbstractTOccurrenceSearcher.java | 2 +- .../search/InvertedIndexSearchPredicate.java | 2 +- .../search/InvertedListMerger.java | 2 +- .../AbstractLSMRTreeDataflowHelper.java | 109 ---- .../dataflow/ExternalRTreeDataflowHelper.java | 108 ---- .../ExternalRTreeDataflowHelperFactory.java | 62 -- .../dataflow/ExternalRTreeLocalResource.java | 78 +++ .../ExternalRTreeLocalResourceFactory.java | 67 +++ .../rtree/dataflow/LSMRTreeDataflowHelper.java | 93 --- .../dataflow/LSMRTreeDataflowHelperFactory.java | 82 --- .../rtree/dataflow/LSMRTreeLocalResource.java | 94 +++ .../dataflow/LSMRTreeLocalResourceFactory.java | 83 +++ .../LSMRTreeWithAntiMatterLocalResource.java | 89 +++ ...RTreeWithAntiMatterLocalResourceFactory.java | 78 +++ ...RTreeWithAntiMatterTuplesDataflowHelper.java | 75 --- ...thAntiMatterTuplesDataflowHelperFactory.java | 79 --- .../am/lsm/rtree/impls/AbstractLSMRTree.java | 8 +- .../am/lsm/rtree/impls/ExternalRTree.java | 24 +- .../lsm/rtree/impls/ExternalRTreeOpContext.java | 8 +- .../storage/am/lsm/rtree/impls/LSMRTree.java | 12 +- .../lsm/rtree/impls/LSMRTreeAbstractCursor.java | 6 +- .../rtree/impls/LSMRTreeCursorInitialState.java | 6 +- .../LSMRTreeDeletedKeysBTreeMergeCursor.java | 8 +- .../am/lsm/rtree/impls/LSMRTreeOpContext.java | 8 +- .../lsm/rtree/impls/LSMRTreeSearchCursor.java | 4 +- .../lsm/rtree/impls/LSMRTreeSortedCursor.java | 9 +- .../impls/LSMRTreeWithAntiMatterTuples.java | 16 +- ...LSMRTreeWithAntiMatterTuplesFlushCursor.java | 6 +- ...SMRTreeWithAntiMatterTuplesSearchCursor.java | 8 +- .../am/lsm/rtree/impls/TreeTupleSorter.java | 6 +- .../am/lsm/rtree/utils/LSMRTreeUtils.java | 26 +- .../storage/am/rtree/api/IRTreeFrame.java | 2 +- .../am/rtree/api/IRTreeInteriorFrame.java | 2 +- .../storage/am/rtree/api/IRTreeLeafFrame.java | 2 +- .../storage/am/rtree/api/IRTreePolicy.java | 2 +- .../am/rtree/dataflow/RTreeDataflowHelper.java | 57 -- .../dataflow/RTreeDataflowHelperFactory.java | 50 -- .../am/rtree/dataflow/RTreeResource.java | 72 +++ .../am/rtree/dataflow/RTreeResourceFactory.java | 58 ++ .../dataflow/RTreeSearchOperatorDescriptor.java | 70 +-- .../RTreeSearchOperatorNodePushable.java | 21 +- .../am/rtree/frames/RStarTreePolicy.java | 2 +- .../am/rtree/frames/RTreeComputationUtils.java | 2 +- .../storage/am/rtree/frames/RTreeNSMFrame.java | 2 +- .../am/rtree/frames/RTreeNSMInteriorFrame.java | 2 +- .../am/rtree/frames/RTreeNSMLeafFrame.java | 2 +- .../storage/am/rtree/frames/RTreePolicy.java | 2 +- .../hyracks/storage/am/rtree/impls/RTree.java | 17 +- .../am/rtree/impls/RTreeCursorInitialState.java | 6 +- .../storage/am/rtree/impls/RTreeOpContext.java | 4 +- .../am/rtree/impls/RTreeSearchCursor.java | 6 +- .../storage/am/rtree/impls/SearchPredicate.java | 2 +- .../am/rtree/impls/UnorderedSlotManager.java | 2 +- .../storage/am/rtree/util/RTreeUtils.java | 2 +- .../hyracks/hyracks-storage-common/pom.xml | 5 + .../storage/common/ICursorInitialState.java | 36 ++ .../apache/hyracks/storage/common/IIndex.java | 139 +++++ .../hyracks/storage/common/IIndexAccessor.java | 105 ++++ .../storage/common/IIndexBulkLoader.java | 51 ++ .../hyracks/storage/common/IIndexCursor.java | 81 +++ .../common/ILocalResourceRepository.java | 32 + .../common/IModificationOperationCallback.java | 52 ++ .../hyracks/storage/common/IResource.java | 31 + .../storage/common/IResourceFactory.java | 34 ++ .../common/IResourceLifecycleManager.java | 82 +++ .../common/ISearchOperationCallback.java | 79 +++ .../storage/common/ISearchPredicate.java | 40 ++ .../hyracks/storage/common/IStorageManager.java | 28 +- .../hyracks/storage/common/LocalResource.java | 105 ++++ .../hyracks/storage/common/MultiComparator.java | 134 +++++ .../storage/common/SingleComparator.java | 43 ++ .../TransientLocalResourceRepository.java | 66 +++ .../common/file/ILocalResourceFactory.java | 24 - .../file/ILocalResourceFactoryProvider.java | 25 - .../common/file/ILocalResourceRepository.java | 32 - .../file/ILocalResourceRepositoryFactory.java | 1 + .../storage/common/file/LocalResource.java | 123 ---- .../file/NoOpLocalResourceFactoryProvider.java | 31 - .../common/file/ResourceIdFactoryProvider.java | 1 + .../file/TransientLocalResourceFactory.java | 27 - .../TransientLocalResourceFactoryProvider.java | 29 - .../file/TransientLocalResourceRepository.java | 66 --- ...TransientLocalResourceRepositoryFactory.java | 2 + .../hyracks/hyracks-test-support/pom.xml | 6 - ...stractModificationOperationCallbackTest.java | 4 +- .../am/btree/AbstractOperationCallbackTest.java | 4 +- .../AbstractSearchOperationCallbackTest.java | 6 +- .../am/btree/OrderedIndexExamplesTest.java | 8 +- .../am/btree/OrderedIndexMultiThreadTest.java | 2 +- .../am/btree/OrderedIndexTestContext.java | 2 +- .../storage/am/btree/OrderedIndexTestUtils.java | 6 +- .../am/common/AbstractIndexLifecycleTest.java | 2 +- .../am/common/AbstractIndexTestWorker.java | 6 +- .../storage/am/common/IIndexTestContext.java | 4 +- .../am/common/IIndexTestWorkerFactory.java | 2 +- .../am/common/ITreeIndexTestContext.java | 2 +- .../am/common/IndexMultiThreadTestDriver.java | 2 +- .../storage/am/common/IndexTestContext.java | 4 +- .../am/common/TestOperationCallback.java | 4 +- .../storage/am/common/TreeIndexTestUtils.java | 4 +- .../am/rtree/AbstractRTreeExamplesTest.java | 6 +- .../storage/am/rtree/RTreeTestUtils.java | 4 +- .../TestIndexLifecycleManagerProvider.java | 35 -- .../test/support/TestStorageManager.java | 26 +- .../TestStorageManagerComponentHolder.java | 24 +- .../hyracks/test/support/TestTaskContext.java | 5 +- .../apache/hyracks/test/support/TestUtils.java | 4 +- .../support/TestVirtualBufferCacheProvider.java | 10 +- .../hyracks-storage-am-bloomfilter-test/pom.xml | 5 - .../storage/am/bloomfilter/BloomFilterTest.java | 2 +- .../util/BloomFilterTestHarness.java | 6 +- .../storage/am/btree/BTreeSearchCursorTest.java | 2 +- .../storage/am/btree/FieldPrefixNSMTest.java | 2 +- .../am/btree/multithread/BTreeTestWorker.java | 4 +- .../multithread/BTreeTestWorkerFactory.java | 2 +- .../storage/am/btree/util/BTreeTestHarness.java | 10 +- .../am/lsm/btree/LSMBTreeExamplesTest.java | 2 +- .../btree/LSMBTreeFilterMergeTestDriver.java | 25 +- .../am/lsm/btree/LSMBTreeMergeTestDriver.java | 8 +- ...MBTreeModificationOperationCallbackTest.java | 14 +- .../LSMBTreeSearchOperationCallbackTest.java | 8 +- .../btree/multithread/LSMBTreeTestWorker.java | 11 +- .../multithread/LSMBTreeTestWorkerFactory.java | 2 +- .../am/lsm/btree/perf/BTreeBulkLoadRunner.java | 2 +- .../storage/am/lsm/btree/perf/BTreeRunner.java | 4 +- .../btree/perf/ConcurrentSkipListRunner.java | 2 +- .../am/lsm/btree/perf/InMemorySortRunner.java | 2 +- .../am/lsm/btree/perf/LSMTreeRunner.java | 12 +- .../am/lsm/btree/util/LSMBTreeTestHarness.java | 15 +- .../am/lsm/common/LSMIndexFileManagerTest.java | 2 +- .../LSMInvertedIndexMergeTest.java | 6 +- .../PartitionedLSMInvertedIndexMergeTest.java | 6 +- .../common/AbstractInvertedIndexDeleteTest.java | 2 +- .../common/AbstractInvertedIndexLoadTest.java | 2 +- .../common/AbstractInvertedIndexSearchTest.java | 2 +- .../common/AbstractInvertedIndexTest.java | 2 +- .../common/LSMInvertedIndexTestHarness.java | 12 +- .../LSMInvertedIndexMultiThreadTestDriver.java | 2 +- .../multithread/LSMInvertedIndexTestWorker.java | 9 +- .../LSMInvertedIndexWorkerFactory.java | 2 +- .../util/LSMInvertedIndexTestContext.java | 2 +- .../util/LSMInvertedIndexTestUtils.java | 6 +- .../lsm/rtree/AbstractLSMRTreeExamplesTest.java | 2 +- .../am/lsm/rtree/LSMRTreeMergeTestDriver.java | 4 +- .../multithread/AbstractLSMRTreeTestWorker.java | 4 +- .../rtree/multithread/LSMRTreeTestWorker.java | 9 +- .../multithread/LSMRTreeTestWorkerFactory.java | 2 +- .../LSMRTreeWithAntiMatterTuplesTestWorker.java | 8 +- ...eeWithAntiMatterTuplesTestWorkerFactory.java | 2 +- .../am/lsm/rtree/util/LSMRTreeTestHarness.java | 12 +- .../storage/am/rtree/RTreeSearchCursorTest.java | 2 +- .../am/rtree/multithread/RTreeTestWorker.java | 6 +- .../multithread/RTreeTestWorkerFactory.java | 2 +- .../am/rtree/utils/RTreeTestHarness.java | 4 +- .../common/BufferCacheRegressionTest.java | 5 +- .../hyracks/storage/common/BufferCacheTest.java | 15 +- 572 files changed, 7413 insertions(+), 10646 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/asterixdb/asterix-algebra/pom.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/pom.xml b/asterixdb/asterix-algebra/pom.xml index e042d97..04f8d63 100644 --- a/asterixdb/asterix-algebra/pom.xml +++ b/asterixdb/asterix-algebra/pom.xml @@ -276,10 +276,6 @@ </dependency> <dependency> <groupId>org.apache.hyracks</groupId> - <artifactId>hyracks-storage-am-lsm-common</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hyracks</groupId> <artifactId>hyracks-api</artifactId> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java index 4e3ae55..50c762e 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java @@ -18,25 +18,16 @@ */ package org.apache.asterix.algebra.operators.physical; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import org.apache.asterix.common.dataflow.ICcApplicationContext; import org.apache.asterix.metadata.MetadataException; import org.apache.asterix.metadata.MetadataManager; import org.apache.asterix.metadata.declared.DataSourceId; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.metadata.entities.Index; -import org.apache.asterix.metadata.utils.DatasetUtil; import org.apache.asterix.om.base.IAObject; import org.apache.asterix.om.constants.AsterixConstantValue; import org.apache.asterix.om.functions.BuiltinFunctions; -import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; -import org.apache.asterix.om.types.IAType; -import org.apache.asterix.om.utils.NonTaggedFormatUtil; import org.apache.asterix.optimizer.rules.am.InvertedIndexAccessMethod; import org.apache.asterix.optimizer.rules.am.InvertedIndexAccessMethod.SearchModifierType; import org.apache.asterix.optimizer.rules.am.InvertedIndexJobGenParams; @@ -49,7 +40,6 @@ import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator; import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag; import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag; -import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable; import org.apache.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag; import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; import org.apache.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue; @@ -57,21 +47,16 @@ import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvir import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestMapOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema; -import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities; import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain; import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext; import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenHelper; import org.apache.hyracks.api.dataflow.IOperatorDescriptor; -import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; -import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.api.dataflow.value.RecordDescriptor; import org.apache.hyracks.api.job.JobSpecification; -import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory; -import org.apache.hyracks.data.std.primitive.ShortPointable; import org.apache.hyracks.dataflow.std.file.IFileSplitProvider; import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; +import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory; import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearchModifierFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexSearchOperatorDescriptor; import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory; @@ -154,87 +139,15 @@ public class InvertedIndexPOperator extends IndexSearchPOperator { throws AlgebricksException { try { IAObject simThresh = ((AsterixConstantValue) similarityThreshold).getObject(); - IAType itemType = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(), - dataset.getItemTypeDataverseName(), dataset.getItemTypeName()).getDatatype(); - int numPrimaryKeys = DatasetUtil.getPartitioningKeys(dataset).size(); + int numPrimaryKeys = dataset.getPrimaryKeys().size(); Index secondaryIndex = MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(), dataset.getDataverseName(), dataset.getDatasetName(), indexName); if (secondaryIndex == null) { throw new AlgebricksException( "Code generation error: no index " + indexName + " for dataset " + datasetName); } - List<List<String>> secondaryKeyFieldEntries = secondaryIndex.getKeyFieldNames(); - List<IAType> secondaryKeyTypeEntries = secondaryIndex.getKeyFieldTypes(); - int numSecondaryKeys = secondaryKeyFieldEntries.size(); - if (numSecondaryKeys != 1) { - throw new AlgebricksException( - "Cannot use " + numSecondaryKeys + " fields as a key for an inverted index. " - + "There can be only one field as a key for the inverted index index."); - } - if (itemType.getTypeTag() != ATypeTag.OBJECT) { - throw new AlgebricksException("Only record types can be indexed."); - } - ARecordType recordType = (ARecordType) itemType; - Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(secondaryKeyTypeEntries.get(0), - secondaryKeyFieldEntries.get(0), recordType); - IAType secondaryKeyType = keyPairType.first; - if (secondaryKeyType == null) { - throw new AlgebricksException( - "Could not find field " + secondaryKeyFieldEntries.get(0) + " in the schema."); - } - - // TODO: For now we assume the type of the generated tokens is the - // same as the indexed field. - // We need a better way of expressing this because tokens may be - // hashed, or an inverted-index may index a list type, etc. - int numTokenKeys = (!isPartitioned) ? numSecondaryKeys : numSecondaryKeys + 1; - ITypeTraits[] tokenTypeTraits = new ITypeTraits[numTokenKeys]; - IBinaryComparatorFactory[] tokenComparatorFactories = new IBinaryComparatorFactory[numTokenKeys]; - for (int i = 0; i < numSecondaryKeys; i++) { - tokenComparatorFactories[i] = NonTaggedFormatUtil.getTokenBinaryComparatorFactory(secondaryKeyType); - tokenTypeTraits[i] = NonTaggedFormatUtil.getTokenTypeTrait(secondaryKeyType); - } - if (isPartitioned) { - // The partitioning field is hardcoded to be a short *without* an Asterix type tag. - tokenComparatorFactories[numSecondaryKeys] = - PointableBinaryComparatorFactory.of(ShortPointable.FACTORY); - tokenTypeTraits[numSecondaryKeys] = ShortPointable.TYPE_TRAITS; - } - IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(unnestMap); - List<LogicalVariable> outputVars = unnestMap.getVariables(); - if (retainInput) { - outputVars = new ArrayList<>(); - VariableUtilities.getLiveVariables(unnestMap, outputVars); - } RecordDescriptor outputRecDesc = JobGenHelper.mkRecordDescriptor(typeEnv, opSchema, context); - - int start = outputRecDesc.getFieldCount() - numPrimaryKeys; - IBinaryComparatorFactory[] invListsComparatorFactories = JobGenHelper - .variablesToAscBinaryComparatorFactories(outputVars, start, numPrimaryKeys, typeEnv, context); - ITypeTraits[] invListsTypeTraits = - JobGenHelper.variablesToTypeTraits(outputVars, start, numPrimaryKeys, typeEnv, context); - ITypeTraits[] filterTypeTraits = DatasetUtil.computeFilterTypeTraits(dataset, recordType); - int[] filterFields; - int[] invertedIndexFields; - int[] filterFieldsForNonBulkLoadOps; - int[] invertedIndexFieldsForNonBulkLoadOps; - if (filterTypeTraits != null) { - filterFields = new int[1]; - filterFields[0] = numTokenKeys + numPrimaryKeys; - invertedIndexFields = new int[numTokenKeys + numPrimaryKeys]; - for (int k = 0; k < invertedIndexFields.length; k++) { - invertedIndexFields[k] = k; - } - - filterFieldsForNonBulkLoadOps = new int[1]; - filterFieldsForNonBulkLoadOps[0] = numPrimaryKeys + numSecondaryKeys; - invertedIndexFieldsForNonBulkLoadOps = new int[numPrimaryKeys + numSecondaryKeys]; - for (int k = 0; k < invertedIndexFieldsForNonBulkLoadOps.length; k++) { - invertedIndexFieldsForNonBulkLoadOps[k] = k; - } - } - ICcApplicationContext appContext = (ICcApplicationContext) context.getAppContext(); Pair<IFileSplitProvider, AlgebricksPartitionConstraint> secondarySplitsAndConstraint = metadataProvider.getSplitProviderAndConstraints(dataset, indexName); // TODO: Here we assume there is only one search key field. @@ -244,25 +157,16 @@ public class InvertedIndexPOperator extends IndexSearchPOperator { InvertedIndexAccessMethod.getSearchModifierFactory(searchModifierType, simThresh, secondaryIndex); IBinaryTokenizerFactory queryTokenizerFactory = InvertedIndexAccessMethod .getBinaryTokenizerFactory(searchModifierType, searchKeyType, secondaryIndex); - ARecordType metaType = dataset.hasMetaPart() - ? (ARecordType) metadataProvider - .findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName()).getType() - : null; - Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = - DatasetUtil.getMergePolicyFactory(dataset, metadataProvider.getMetadataTxnContext()); - IIndexDataflowHelperFactory dataflowHelperFactory = dataset.getIndexDataflowHelperFactory(metadataProvider, - secondaryIndex, recordType, metaType, compactionInfo.first, compactionInfo.second); + IIndexDataflowHelperFactory dataflowHelperFactory = + new IndexDataflowHelperFactory(metadataProvider.getStorageComponentProvider().getStorageManager(), + secondarySplitsAndConstraint.first); LSMInvertedIndexSearchOperatorDescriptor invIndexSearchOp = new LSMInvertedIndexSearchOperatorDescriptor( - jobSpec, queryField, appContext.getStorageManager(), secondarySplitsAndConstraint.first, - appContext.getIndexLifecycleManagerProvider(), tokenTypeTraits, tokenComparatorFactories, - invListsTypeTraits, invListsComparatorFactories, dataflowHelperFactory, queryTokenizerFactory, - searchModifierFactory, outputRecDesc, retainInput, retainMissing, context.getMissingWriterFactory(), + jobSpec, outputRecDesc, queryField, dataflowHelperFactory, queryTokenizerFactory, + searchModifierFactory, retainInput, retainMissing, context.getMissingWriterFactory(), dataset.getSearchCallbackFactory(metadataProvider.getStorageComponentProvider(), secondaryIndex, ((JobEventListenerFactory) jobSpec.getJobletEventListenerFactory()).getJobId(), IndexOperation.SEARCH, null), - minFilterFieldIndexes, maxFilterFieldIndexes, - metadataProvider.getStorageComponentProvider().getMetadataPageManagerFactory(), - isFullTextSearchQuery); + minFilterFieldIndexes, maxFilterFieldIndexes, isFullTextSearchQuery, numPrimaryKeys, false); return new Pair<>(invIndexSearchOp, secondarySplitsAndConstraint.second); } catch (MetadataException e) { throw new AlgebricksException(e); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveUnusedOneToOneEquiJoinRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveUnusedOneToOneEquiJoinRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveUnusedOneToOneEquiJoinRule.java index a20a33d..6a70786 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveUnusedOneToOneEquiJoinRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveUnusedOneToOneEquiJoinRule.java @@ -26,7 +26,6 @@ import java.util.Set; import org.apache.asterix.metadata.declared.DatasetDataSource; import org.apache.asterix.metadata.entities.InternalDatasetDetails; -import org.apache.asterix.metadata.utils.DatasetUtil; import org.apache.commons.lang3.mutable.Mutable; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; @@ -231,7 +230,7 @@ public class RemoveUnusedOneToOneEquiJoinRule implements IAlgebraicRewriteRule { DatasetDataSource datasetDataSource = (DatasetDataSource) dataScan.getDataSource(); pkVars.clear(); if (datasetDataSource.getDataset().getDatasetDetails() instanceof InternalDatasetDetails) { - int numPKs = DatasetUtil.getPartitioningKeys(datasetDataSource.getDataset()).size(); + int numPKs = datasetDataSource.getDataset().getPrimaryKeys().size(); for (int i = 0; i < numPKs; i++) { pkVars.add(dataScan.getVariables().get(i)); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java index 46b421b..7f35d08 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java @@ -37,7 +37,6 @@ import org.apache.asterix.metadata.entities.FeedConnection; import org.apache.asterix.metadata.entities.FeedPolicyEntity; import org.apache.asterix.metadata.entities.InternalDatasetDetails; import org.apache.asterix.metadata.feeds.BuiltinFeedPolicies; -import org.apache.asterix.metadata.utils.DatasetUtil; import org.apache.asterix.om.base.AString; import org.apache.asterix.om.constants.AsterixConstantValue; import org.apache.asterix.om.functions.BuiltinFunctions; @@ -124,7 +123,7 @@ public class UnnestToDataScanRule implements IAlgebraicRewriteRule { DataSourceId asid = new DataSourceId(dataverseName, datasetName); List<LogicalVariable> variables = new ArrayList<>(); if (dataset.getDatasetType() == DatasetType.INTERNAL) { - int numPrimaryKeys = DatasetUtil.getPartitioningKeys(dataset).size(); + int numPrimaryKeys = dataset.getPrimaryKeys().size(); for (int i = 0; i < numPrimaryKeys; i++) { variables.add(context.newVar()); } @@ -144,8 +143,8 @@ public class UnnestToDataScanRule implements IAlgebraicRewriteRule { // Adds equivalence classes --- one equivalent class between a primary key // variable and a record field-access expression. IAType[] schemaTypes = dataSource.getSchemaTypes(); - ARecordType recordType = (ARecordType) (hasMeta ? schemaTypes[schemaTypes.length - 2] - : schemaTypes[schemaTypes.length - 1]); + ARecordType recordType = + (ARecordType) (hasMeta ? schemaTypes[schemaTypes.length - 2] : schemaTypes[schemaTypes.length - 1]); ARecordType metaRecordType = (ARecordType) (hasMeta ? schemaTypes[schemaTypes.length - 1] : null); EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(scan, variables, recordType, metaRecordType, dataset, context); @@ -198,7 +197,7 @@ public class UnnestToDataScanRule implements IAlgebraicRewriteRule { private void addPrimaryKey(List<LogicalVariable> scanVariables, DataSource dataSource, IOptimizationContext context) { List<LogicalVariable> primaryKey = dataSource.getPrimaryKeyVariables(scanVariables); - List<LogicalVariable> tail = new ArrayList<LogicalVariable>(); + List<LogicalVariable> tail = new ArrayList<>(); tail.addAll(scanVariables); FunctionalDependency pk = new FunctionalDependency(primaryKey, tail); context.addPrimaryKey(pk); @@ -215,8 +214,8 @@ public class UnnestToDataScanRule implements IAlgebraicRewriteRule { Dataset dataset = metadataProvider.findDataset(aqlId.getDataverseName(), targetDataset); ARecordType feedOutputType = (ARecordType) metadataProvider.findType(aqlId.getDataverseName(), outputType); Feed sourceFeed = metadataProvider.findFeed(aqlId.getDataverseName(), sourceFeedName); - FeedConnection feedConnection = metadataProvider.findFeedConnection(aqlId.getDataverseName(), sourceFeedName, - targetDataset); + FeedConnection feedConnection = + metadataProvider.findFeedConnection(aqlId.getDataverseName(), sourceFeedName, targetDataset); ARecordType metaType = null; // Does dataset have meta? if (dataset.hasMetaPart()) { @@ -263,8 +262,8 @@ public class UnnestToDataScanRule implements IAlgebraicRewriteRule { } FeedDataSource feedDataSource = new FeedDataSource(sourceFeed, aqlId, targetDataset, feedOutputType, metaType, pkTypes, partitioningKeys, keyAccessScalarFunctionCallExpression, sourceFeed.getFeedId(), - FeedRuntimeType.valueOf(subscriptionLocation), locations.split(","), - context.getComputationNodeDomain(), feedConnection); + FeedRuntimeType.valueOf(subscriptionLocation), locations.split(","), context.getComputationNodeDomain(), + feedConnection); feedDataSource.getProperties().put(BuiltinFeedPolicies.CONFIG_FEED_POLICY_KEY, feedPolicy); return feedDataSource; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java index 86d2b9f..8df1756 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java @@ -30,7 +30,6 @@ import org.apache.asterix.common.config.DatasetConfig.IndexType; import org.apache.asterix.dataflow.data.common.ExpressionTypeComputer; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.metadata.entities.Index; -import org.apache.asterix.metadata.utils.DatasetUtil; import org.apache.asterix.metadata.utils.MetadataUtil; import org.apache.asterix.om.base.AOrderedList; import org.apache.asterix.om.base.AString; @@ -643,7 +642,7 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew List<List<String>> subTreePKs = null; if (!fromAdditionalDataSource) { - subTreePKs = DatasetUtil.getPartitioningKeys(subTree.getDataset()); + subTreePKs = subTree.getDataset().getPrimaryKeys(); // Check whether this variable is PK, not a record variable. if (varIndex <= subTreePKs.size() - 1) { fieldName = subTreePKs.get(varIndex); @@ -654,8 +653,7 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew // Need to check additional dataset one by one for (int i = 0; i < subTree.getIxJoinOuterAdditionalDatasets().size(); i++) { if (subTree.getIxJoinOuterAdditionalDatasets().get(i) != null) { - subTreePKs = - DatasetUtil.getPartitioningKeys(subTree.getIxJoinOuterAdditionalDatasets().get(i)); + subTreePKs = subTree.getIxJoinOuterAdditionalDatasets().get(i).getPrimaryKeys(); // Check whether this variable is PK, not a record variable. if (subTreePKs.contains(var) && varIndex <= subTreePKs.size() - 1) { @@ -698,10 +696,10 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew * * @throws AlgebricksException */ - protected List<String> getFieldNameFromSubTree(IOptimizableFuncExpr optFuncExpr, - OptimizableOperatorSubTree subTree, int opIndex, int assignVarIndex, ARecordType recordType, - int funcVarIndex, ILogicalExpression parentFuncExpr, LogicalVariable recordVar, ARecordType metaType, - LogicalVariable metaVar) throws AlgebricksException { + protected List<String> getFieldNameFromSubTree(IOptimizableFuncExpr optFuncExpr, OptimizableOperatorSubTree subTree, + int opIndex, int assignVarIndex, ARecordType recordType, int funcVarIndex, + ILogicalExpression parentFuncExpr, LogicalVariable recordVar, ARecordType metaType, LogicalVariable metaVar) + throws AlgebricksException { // Get expression corresponding to opVar at varIndex. AbstractLogicalExpression expr = null; AbstractFunctionCallExpression childFuncExpr = null; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java index c3b6473..59cb373 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java @@ -36,7 +36,6 @@ import org.apache.asterix.metadata.declared.DataSourceId; import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.metadata.entities.ExternalDatasetDetails; import org.apache.asterix.metadata.entities.Index; -import org.apache.asterix.metadata.utils.DatasetUtil; import org.apache.asterix.metadata.utils.KeyFieldTypeUtil; import org.apache.asterix.om.base.ABoolean; import org.apache.asterix.om.base.AInt32; @@ -190,8 +189,7 @@ public class AccessMethodUtils { private static void constructNewOptFuncExprAndAddToAnalysisCtx(AbstractFunctionCallExpression funcExpr, LogicalVariable fieldVar, ILogicalExpression expression, IAType expressionType, AccessMethodAnalysisContext analysisCtx) { - OptimizableFuncExpr newOptFuncExpr = - new OptimizableFuncExpr(funcExpr, fieldVar, expression, expressionType); + OptimizableFuncExpr newOptFuncExpr = new OptimizableFuncExpr(funcExpr, fieldVar, expression, expressionType); addNewOptFuncExprToAnalysisCtx(funcExpr, newOptFuncExpr, analysisCtx); } @@ -217,8 +215,7 @@ public class AccessMethodUtils { /** * Fetches each element and calls the check for the type and value in the given list using the given cursor. */ - private static void checkEachElementInFTSearchListPredicate(IACursor oListCursor) - throws AlgebricksException { + private static void checkEachElementInFTSearchListPredicate(IACursor oListCursor) throws AlgebricksException { String argValue; IAObject element; while (oListCursor.next()) { @@ -331,7 +328,7 @@ public class AccessMethodUtils { numPrimaryKeys = IndexingConstants .getRIDSize(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties()); } else { - numPrimaryKeys = DatasetUtil.getPartitioningKeys(dataset).size(); + numPrimaryKeys = dataset.getPrimaryKeys().size(); } int numSecondaryKeys = KeyFieldTypeUtil.getNumSecondaryKeys(index, recordType, metaRecordType); int numVars = (primaryKeysOnly) ? numPrimaryKeys : numPrimaryKeys + numSecondaryKeys; @@ -347,7 +344,7 @@ public class AccessMethodUtils { numPrimaryKeys = IndexingConstants .getRIDSize(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties()); } else { - numPrimaryKeys = DatasetUtil.getPartitioningKeys(dataset).size(); + numPrimaryKeys = dataset.getPrimaryKeys().size(); } List<LogicalVariable> primaryKeyVars = new ArrayList<>(); List<LogicalVariable> sourceVars = null; @@ -365,7 +362,7 @@ public class AccessMethodUtils { public static List<LogicalVariable> getPrimaryKeyVarsFromPrimaryUnnestMap(Dataset dataset, ILogicalOperator unnestMapOp) { - int numPrimaryKeys = DatasetUtil.getPartitioningKeys(dataset).size(); + int numPrimaryKeys = dataset.getPrimaryKeys().size(); List<LogicalVariable> primaryKeyVars = new ArrayList<>(); List<LogicalVariable> sourceVars = null; @@ -495,8 +492,8 @@ public class AccessMethodUtils { secondaryIndexOutputTypes); // An index search is expressed as an unnest over an index-search function. IFunctionInfo secondaryIndexSearch = FunctionUtil.getFunctionInfo(BuiltinFunctions.INDEX_SEARCH); - UnnestingFunctionCallExpression secondaryIndexSearchFunc = new UnnestingFunctionCallExpression( - secondaryIndexSearch, secondaryIndexFuncArgs); + UnnestingFunctionCallExpression secondaryIndexSearchFunc = + new UnnestingFunctionCallExpression(secondaryIndexSearch, secondaryIndexFuncArgs); secondaryIndexSearchFunc.setReturnsUniqueValues(true); // This is the operator that jobgen will be looking for. It contains an unnest function that has all necessary arguments to determine // which index to use, which variables contain the index-search keys, what is the original dataset, etc. @@ -532,17 +529,15 @@ public class AccessMethodUtils { Dataset dataset, ARecordType recordType, ARecordType metaRecordType, ILogicalOperator inputOp, IOptimizationContext context, boolean sortPrimaryKeys, boolean retainInput, boolean retainNull, boolean requiresBroadcast) throws AlgebricksException { - List<LogicalVariable> primaryKeyVars = AccessMethodUtils.getPrimaryKeyVarsFromSecondaryUnnestMap(dataset, - inputOp); + List<LogicalVariable> primaryKeyVars = + AccessMethodUtils.getPrimaryKeyVarsFromSecondaryUnnestMap(dataset, inputOp); // Optionally add a sort on the primary-index keys before searching the primary index. OrderOperator order = null; if (sortPrimaryKeys) { order = new OrderOperator(); for (LogicalVariable pkVar : primaryKeyVars) { - Mutable<ILogicalExpression> vRef = new MutableObject<>( - new VariableReferenceExpression(pkVar)); - order.getOrderExpressions() - .add(new Pair<>(OrderOperator.ASC_ORDER, vRef)); + Mutable<ILogicalExpression> vRef = new MutableObject<>(new VariableReferenceExpression(pkVar)); + order.getOrderExpressions().add(new Pair<>(OrderOperator.ASC_ORDER, vRef)); } // The secondary-index search feeds into the sort. order.getInputs().add(new MutableObject<>(inputOp)); @@ -568,8 +563,8 @@ public class AccessMethodUtils { appendPrimaryIndexTypes(dataset, recordType, metaRecordType, primaryIndexOutputTypes); // An index search is expressed as an unnest over an index-search function. IFunctionInfo primaryIndexSearch = FunctionUtil.getFunctionInfo(BuiltinFunctions.INDEX_SEARCH); - AbstractFunctionCallExpression primaryIndexSearchFunc = new ScalarFunctionCallExpression(primaryIndexSearch, - primaryIndexFuncArgs); + AbstractFunctionCallExpression primaryIndexSearchFunc = + new ScalarFunctionCallExpression(primaryIndexSearch, primaryIndexFuncArgs); // This is the operator that jobgen will be looking for. It contains an unnest function that has all necessary arguments to determine // which index to use, which variables contain the index-search keys, what is the original dataset, etc. AbstractUnnestMapOperator primaryIndexUnnestOp = null; @@ -613,14 +608,14 @@ public class AccessMethodUtils { if (selectOp.getCondition().getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) { if (((AbstractFunctionCallExpression) selectOp.getCondition().getValue()).getFunctionIdentifier() .equals(AlgebricksBuiltinFunctions.NOT)) { - ScalarFunctionCallExpression notFuncExpr = (ScalarFunctionCallExpression) selectOp - .getCondition().getValue(); + ScalarFunctionCallExpression notFuncExpr = + (ScalarFunctionCallExpression) selectOp.getCondition().getValue(); if (notFuncExpr.getArguments().get(0).getValue() .getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) { if (((AbstractFunctionCallExpression) notFuncExpr.getArguments().get(0).getValue()) .getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.IS_MISSING)) { - isNullFuncExpr = (ScalarFunctionCallExpression) notFuncExpr.getArguments().get(0) - .getValue(); + isNullFuncExpr = + (ScalarFunctionCallExpression) notFuncExpr.getArguments().get(0).getValue(); if (isNullFuncExpr.getArguments().get(0).getValue() .getExpressionTag() == LogicalExpressionTag.VARIABLE) { foundSelectNonNull = true; @@ -661,43 +656,40 @@ public class AccessMethodUtils { } private static void appendExternalRecPrimaryKeys(Dataset dataset, List<Object> target) throws AsterixException { - int numPrimaryKeys = IndexingConstants - .getRIDSize(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties()); + int numPrimaryKeys = + IndexingConstants.getRIDSize(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties()); for (int i = 0; i < numPrimaryKeys; i++) { target.add(IndexingConstants.getFieldType(i)); } } private static void writeVarList(List<LogicalVariable> varList, List<Mutable<ILogicalExpression>> funcArgs) { - Mutable<ILogicalExpression> numKeysRef = new MutableObject<>( - new ConstantExpression(new AsterixConstantValue(new AInt32(varList.size())))); + Mutable<ILogicalExpression> numKeysRef = + new MutableObject<>(new ConstantExpression(new AsterixConstantValue(new AInt32(varList.size())))); funcArgs.add(numKeysRef); for (LogicalVariable keyVar : varList) { - Mutable<ILogicalExpression> keyVarRef = new MutableObject<>( - new VariableReferenceExpression(keyVar)); + Mutable<ILogicalExpression> keyVarRef = new MutableObject<>(new VariableReferenceExpression(keyVar)); funcArgs.add(keyVarRef); } } private static void addStringArg(String argument, List<Mutable<ILogicalExpression>> funcArgs) { - Mutable<ILogicalExpression> stringRef = new MutableObject<>( - new ConstantExpression(new AsterixConstantValue(new AString(argument)))); + Mutable<ILogicalExpression> stringRef = + new MutableObject<>(new ConstantExpression(new AsterixConstantValue(new AString(argument)))); funcArgs.add(stringRef); } public static UnnestMapOperator createExternalDataLookupUnnestMap(AbstractDataSourceOperator dataSourceOp, Dataset dataset, ARecordType recordType, ILogicalOperator inputOp, IOptimizationContext context, boolean retainInput, boolean retainNull) throws AlgebricksException { - List<LogicalVariable> primaryKeyVars = AccessMethodUtils.getPrimaryKeyVarsFromSecondaryUnnestMap(dataset, - inputOp); + List<LogicalVariable> primaryKeyVars = + AccessMethodUtils.getPrimaryKeyVarsFromSecondaryUnnestMap(dataset, inputOp); // add a sort on the RID fields before fetching external data. OrderOperator order = new OrderOperator(); for (LogicalVariable pkVar : primaryKeyVars) { - Mutable<ILogicalExpression> vRef = new MutableObject<>( - new VariableReferenceExpression(pkVar)); - order.getOrderExpressions() - .add(new Pair<>(OrderOperator.ASC_ORDER, vRef)); + Mutable<ILogicalExpression> vRef = new MutableObject<>(new VariableReferenceExpression(pkVar)); + order.getOrderExpressions().add(new Pair<>(OrderOperator.ASC_ORDER, vRef)); } // The secondary-index search feeds into the sort. order.getInputs().add(new MutableObject<>(inputOp)); @@ -719,8 +711,8 @@ public class AccessMethodUtils { appendExternalRecTypes(dataset, recordType, outputTypes); IFunctionInfo externalLookup = FunctionUtil.getFunctionInfo(BuiltinFunctions.EXTERNAL_LOOKUP); - AbstractFunctionCallExpression externalLookupFunc = new ScalarFunctionCallExpression(externalLookup, - externalLookupArgs); + AbstractFunctionCallExpression externalLookupFunc = + new ScalarFunctionCallExpression(externalLookup, externalLookupArgs); UnnestMapOperator unnestOp = new UnnestMapOperator(externalUnnestVars, new MutableObject<ILogicalExpression>(externalLookupFunc), outputTypes, retainInput); // Fed by the order operator or the secondaryIndexUnnestOp. @@ -731,8 +723,8 @@ public class AccessMethodUtils { //set the physical operator DataSourceId dataSourceId = new DataSourceId(dataset.getDataverseName(), dataset.getDatasetName()); - unnestOp.setPhysicalOperator(new ExternalDataLookupPOperator(dataSourceId, dataset, recordType, - primaryKeyVars, false, retainInput, retainNull)); + unnestOp.setPhysicalOperator(new ExternalDataLookupPOperator(dataSourceId, dataset, recordType, primaryKeyVars, + false, retainInput, retainNull)); return unnestOp; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java index c7401b2..83b277d 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java @@ -129,11 +129,11 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule { List<Mutable<ILogicalExpression>> assignKeyExprList = new ArrayList<>(); for (IOptimizableFuncExpr optFuncExpr : optFuncExprs) { - ComparisonKind ck = AlgebricksBuiltinFunctions - .getComparisonType(optFuncExpr.getFuncExpr().getFunctionIdentifier()); + ComparisonKind ck = + AlgebricksBuiltinFunctions.getComparisonType(optFuncExpr.getFuncExpr().getFunctionIdentifier()); ILogicalExpression searchKeyExpr = optFuncExpr.getConstantExpr(0); LogicalVariable var = context.newVar(); - assignKeyExprList.add(new MutableObject<ILogicalExpression>(searchKeyExpr)); + assignKeyExprList.add(new MutableObject<>(searchKeyExpr)); assignKeyVarList.add(var); if (ck == ComparisonKind.GE || ck == ComparisonKind.GT) { minFilterVars.add(var); @@ -177,7 +177,7 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule { dataSourceScanOp.setAdditionalFilteringExpressions(additionalFilteringExpressions); assignOp.getInputs() - .add(new MutableObject<ILogicalOperator>(dataSourceScanOp.getInputs().get(0).getValue())); + .add(new MutableObject<>(dataSourceScanOp.getInputs().get(0).getValue())); dataSourceScanOp.getInputs().get(0).setValue(assignOp); } } else if (descendantOp.getOperatorTag() == LogicalOperatorTag.UNNEST_MAP) { @@ -195,8 +195,8 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule { List<LogicalVariable> minFilterVars = new ArrayList<>(); List<LogicalVariable> maxFilterVars = new ArrayList<>(); - AssignOperator assignOp = createAssignOperator(optFuncExprs, minFilterVars, maxFilterVars, - context); + AssignOperator assignOp = + createAssignOperator(optFuncExprs, minFilterVars, maxFilterVars, context); unnestMapOp.setMinFilterVars(minFilterVars); unnestMapOp.setMaxFilterVars(maxFilterVars); @@ -208,7 +208,7 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule { } unnestMapOp.setAdditionalFilteringExpressions(additionalFilteringExpressions); assignOp.getInputs() - .add(new MutableObject<ILogicalOperator>(unnestMapOp.getInputs().get(0).getValue())); + .add(new MutableObject<>(unnestMapOp.getInputs().get(0).getValue())); unnestMapOp.getInputs().get(0).setValue(assignOp); } } @@ -246,8 +246,7 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule { } else { throw new AlgebricksException("Unexpected function for Unnest Map: " + fid); } - return ((MetadataProvider) context.getMetadataProvider()).findDataset(dataverseName, - datasetName); + return ((MetadataProvider) context.getMetadataProvider()).findDataset(dataverseName, datasetName); } } if (descendantOp.getInputs().isEmpty()) { @@ -318,8 +317,8 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule { if (funcVarIndex == -1) { continue; } - List<String> fieldName = getFieldNameFromSubAssignTree(optFuncExpr, descendantOp, varIndex, - recType).second; + List<String> fieldName = + getFieldNameFromSubAssignTree(optFuncExpr, descendantOp, varIndex, recType).second; if (fieldName == null) { return false; } @@ -336,7 +335,7 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule { continue; } // The variable value is one of the partitioning fields. - List<String> fieldName = DatasetUtil.getPartitioningKeys(dataset).get(varIndex); + List<String> fieldName = dataset.getPrimaryKeys().get(varIndex); if (fieldName == null) { return false; } @@ -379,7 +378,7 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule { int numSecondaryKeys = KeyFieldTypeUtil.getNumSecondaryKeys(index, recType, metaRecType); List<String> fieldName; if (varIndex >= numSecondaryKeys) { - fieldName = DatasetUtil.getPartitioningKeys(dataset).get(varIndex - numSecondaryKeys); + fieldName = dataset.getPrimaryKeys().get(varIndex - numSecondaryKeys); } else { fieldName = index.getKeyFieldNames().get(varIndex); } @@ -411,8 +410,7 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule { } AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr; FunctionIdentifier funcIdent = funcExpr.getFunctionIdentifier(); - if (funcIdent == BuiltinFunctions.FIELD_ACCESS_BY_NAME - || funcIdent == BuiltinFunctions.FIELD_ACCESS_BY_INDEX) { + if (funcIdent == BuiltinFunctions.FIELD_ACCESS_BY_NAME || funcIdent == BuiltinFunctions.FIELD_ACCESS_BY_INDEX) { //get the variable from here. Figure out which input it came from. Go to that input!!! ArrayList<LogicalVariable> usedVars = new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java index 18358c6..2534680 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java @@ -26,7 +26,6 @@ import org.apache.asterix.common.exceptions.ErrorCode; import org.apache.asterix.metadata.declared.DataSource; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.metadata.entities.Dataset; -import org.apache.asterix.metadata.utils.DatasetUtil; import org.apache.asterix.om.functions.BuiltinFunctions; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; @@ -84,8 +83,7 @@ public class OptimizableOperatorSubTree { private List<Dataset> ixJoinOuterAdditionalDatasets = null; private List<ARecordType> ixJoinOuterAdditionalRecordTypes = null; - public boolean initFromSubTree(Mutable<ILogicalOperator> subTreeOpRef) - throws AlgebricksException { + public boolean initFromSubTree(Mutable<ILogicalOperator> subTreeOpRef) throws AlgebricksException { reset(); rootRef = subTreeOpRef; root = subTreeOpRef.getValue(); @@ -226,8 +224,8 @@ public class OptimizableOperatorSubTree { Dataset ds = null; ARecordType rType = null; - List<Mutable<ILogicalOperator>> sourceOpRefs = new ArrayList<Mutable<ILogicalOperator>>(); - List<DataSourceType> dsTypes = new ArrayList<DataSourceType>(); + List<Mutable<ILogicalOperator>> sourceOpRefs = new ArrayList<>(); + List<DataSourceType> dsTypes = new ArrayList<>(); sourceOpRefs.add(getDataSourceRef()); dsTypes.add(getDataSourceType()); @@ -247,8 +245,7 @@ public class OptimizableOperatorSubTree { IDataSource<?> datasource = dataSourceScan.getDataSource(); if (datasource instanceof DataSource) { byte dsType = ((DataSource) datasource).getDatasourceType(); - if (dsType != DataSource.Type.INTERNAL_DATASET - && dsType != DataSource.Type.EXTERNAL_DATASET) { + if (dsType != DataSource.Type.INTERNAL_DATASET && dsType != DataSource.Type.EXTERNAL_DATASET) { return false; } } @@ -376,12 +373,12 @@ public class OptimizableOperatorSubTree { */ public void getPrimaryKeyVars(Mutable<ILogicalOperator> dataSourceRefToFetch, List<LogicalVariable> target) throws AlgebricksException { - Mutable<ILogicalOperator> dataSourceRefToFetchKey = (dataSourceRefToFetch == null) ? dataSourceRef - : dataSourceRefToFetch; + Mutable<ILogicalOperator> dataSourceRefToFetchKey = + (dataSourceRefToFetch == null) ? dataSourceRef : dataSourceRefToFetch; switch (dataSourceType) { case DATASOURCE_SCAN: DataSourceScanOperator dataSourceScan = (DataSourceScanOperator) getDataSourceRef().getValue(); - int numPrimaryKeys = DatasetUtil.getPartitioningKeys(getDataset()).size(); + int numPrimaryKeys = dataset.getPrimaryKeys().size(); for (int i = 0; i < numPrimaryKeys; i++) { target.add(dataSourceScan.getVariables().get(i)); } @@ -400,6 +397,7 @@ public class OptimizableOperatorSubTree { } } + public List<LogicalVariable> getDataSourceVariables() throws AlgebricksException { switch (getDataSourceType()) { case DATASOURCE_SCAN: @@ -421,8 +419,8 @@ public class OptimizableOperatorSubTree { case DATASOURCE_SCAN: case EXTERNAL_SCAN: case PRIMARY_INDEX_LOOKUP: - AbstractScanOperator scanOp = (AbstractScanOperator) getIxJoinOuterAdditionalDataSourceRefs() - .get(idx).getValue(); + AbstractScanOperator scanOp = + (AbstractScanOperator) getIxJoinOuterAdditionalDataSourceRefs().get(idx).getValue(); return scanOp.getVariables(); case COLLECTION_SCAN: return new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java index 2662be8..d5e01a4 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java @@ -211,7 +211,7 @@ class LangExpressionToPlanTranslator metadataProvider.findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName()); DatasetDataSource targetDatasource = validateDatasetInfo(metadataProvider, stmt.getDataverseName(), stmt.getDatasetName()); - List<List<String>> partitionKeys = DatasetUtil.getPartitioningKeys(targetDatasource.getDataset()); + List<List<String>> partitionKeys = targetDatasource.getDataset().getPrimaryKeys(); if (dataset.hasMetaPart()) { throw new AlgebricksException( dataset.getDatasetName() + ": load dataset is not supported on Datasets with Meta records"); @@ -361,7 +361,7 @@ class LangExpressionToPlanTranslator ArrayList<LogicalVariable> vars = new ArrayList<>(); ArrayList<Mutable<ILogicalExpression>> exprs = new ArrayList<>(); List<Mutable<ILogicalExpression>> varRefsForLoading = new ArrayList<>(); - List<List<String>> partitionKeys = DatasetUtil.getPartitioningKeys(targetDatasource.getDataset()); + List<List<String>> partitionKeys = targetDatasource.getDataset().getPrimaryKeys(); int numOfPrimaryKeys = partitionKeys.size(); for (int i = 0; i < numOfPrimaryKeys; i++) { if (keySourceIndicator == null || keySourceIndicator.get(i).intValue() == 0) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AppRuntimeContextProviderForRecovery.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AppRuntimeContextProviderForRecovery.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AppRuntimeContextProviderForRecovery.java index 6a3472d..397b459 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AppRuntimeContextProviderForRecovery.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AppRuntimeContextProviderForRecovery.java @@ -27,9 +27,9 @@ import org.apache.asterix.common.transactions.ITransactionSubsystem; import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler; import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker; +import org.apache.hyracks.storage.common.ILocalResourceRepository; import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.file.IFileMapProvider; -import org.apache.hyracks.storage.common.file.ILocalResourceRepository; public class AppRuntimeContextProviderForRecovery implements IAppRuntimeContextProvider { @@ -76,7 +76,7 @@ public class AppRuntimeContextProviderForRecovery implements IAppRuntimeContextP @Override public IIOManager getIOManager() { - return asterixAppRuntimeContext.getIOManager(); + return asterixAppRuntimeContext.getIoManager(); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ConnectorApiServlet.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ConnectorApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ConnectorApiServlet.java index cb3c063..421ee0e 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ConnectorApiServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ConnectorApiServlet.java @@ -34,7 +34,6 @@ import org.apache.asterix.metadata.MetadataManager; import org.apache.asterix.metadata.MetadataTransactionContext; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.metadata.entities.Dataset; -import org.apache.asterix.metadata.utils.DatasetUtil; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.utils.FlushDatasetUtil; import org.apache.hyracks.api.client.IHyracksClientConnection; @@ -110,7 +109,7 @@ public class ConnectorApiServlet extends AbstractServlet { metadataProvider.splitsForDataset(mdTxnCtx, dataverseName, datasetName, datasetName, temp); ARecordType recordType = (ARecordType) metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName()); - List<List<String>> primaryKeys = DatasetUtil.getPartitioningKeys(dataset); + List<List<String>> primaryKeys = dataset.getPrimaryKeys(); StringBuilder pkStrBuf = new StringBuilder(); for (List<String> keys : primaryKeys) { for (String key : keys) {
