Merge pre-rc changes to master Change-Id: I578ffd7ef17784034b14f3c0d23cd5094e39f6e2
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/a0490ba6 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/a0490ba6 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/a0490ba6 Branch: refs/heads/master Commit: a0490ba6f46f28470052b3cadeec1a60be0f394e Parents: 40cff7b e32cb67 Author: Ian Maxon <ima...@apache.org> Authored: Mon Nov 6 15:41:03 2017 -0800 Committer: Ian Maxon <ima...@apache.org> Committed: Mon Nov 6 15:41:03 2017 -0800 ---------------------------------------------------------------------- .../TestLsmBtreeIoOpCallbackFactory.java | 9 +++-- .../common/context/DatasetLifecycleManager.java | 15 ++++---- .../asterix/common/context/DatasetResource.java | 11 +++--- .../AbstractLSMIOOperationCallback.java | 36 +++++++++++++------- .../LSMBTreeIOOperationCallback.java | 5 +-- .../LSMBTreeIOOperationCallbackFactory.java | 5 +-- .../LSMBTreeWithBuddyIOOperationCallback.java | 5 +++ ...TreeWithBuddyIOOperationCallbackFactory.java | 5 +-- .../LSMInvertedIndexIOOperationCallback.java | 5 +-- ...InvertedIndexIOOperationCallbackFactory.java | 5 +-- .../LSMRTreeIOOperationCallback.java | 5 +-- .../LSMRTreeIOOperationCallbackFactory.java | 5 +-- .../LSMBTreeIOOperationCallbackTest.java | 11 +++--- ...SMBTreeWithBuddyIOOperationCallbackTest.java | 11 +++--- ...LSMInvertedIndexIOOperationCallbackTest.java | 11 +++--- .../LSMRTreeIOOperationCallbackTest.java | 11 +++--- .../am/common/impls/AbstractTreeIndex.java | 5 --- .../dataflow/ExternalBTreeLocalResource.java | 2 +- .../ExternalBTreeWithBuddyLocalResource.java | 2 +- .../btree/dataflow/LSMBTreeLocalResource.java | 4 +-- .../am/lsm/btree/impls/ExternalBTree.java | 12 +++---- .../lsm/btree/impls/ExternalBTreeWithBuddy.java | 10 ++---- .../storage/am/lsm/btree/impls/LSMBTree.java | 20 +++++------ .../btree/impls/LSMBTreeMemoryComponent.java | 5 +-- .../impls/LSMBTreeWithBuddyMemoryComponent.java | 9 ++--- .../am/lsm/btree/utils/LSMBTreeUtil.java | 19 ++++++----- .../AbstractLSMWithBuddyMemoryComponent.java | 6 ++-- .../lsm/common/api/ILSMIOOperationCallback.java | 16 ++++++++- .../api/ILSMIOOperationCallbackFactory.java | 2 +- .../storage/am/lsm/common/api/ILSMIndex.java | 5 +++ .../lsm/common/impls/AbstractLSMComponent.java | 4 ++- .../common/impls/AbstractLSMDiskComponent.java | 4 +-- .../am/lsm/common/impls/AbstractLSMIndex.java | 17 ++++----- .../impls/AbstractLSMMemoryComponent.java | 6 ++-- .../BlockingIOOperationCallbackWrapper.java | 10 ++++-- .../storage/am/lsm/common/impls/LSMHarness.java | 13 ++++--- .../impls/NoOpIOOperationCallbackFactory.java | 12 +++++-- .../common/impls/StubIOOperationCallback.java | 20 +++++++---- .../lsm/common/impls/ThreadCountingTracker.java | 2 +- .../dataflow/LSMInvertedIndexLocalResource.java | 12 +++---- .../invertedindex/impls/LSMInvertedIndex.java | 11 +++--- .../impls/LSMInvertedIndexMemoryComponent.java | 8 ++--- .../impls/PartitionedLSMInvertedIndex.java | 14 ++++---- .../inmemory/InMemoryInvertedIndex.java | 5 --- .../ondisk/OnDiskInvertedIndex.java | 5 --- .../invertedindex/util/InvertedIndexUtils.java | 22 ++++++------ .../dataflow/ExternalRTreeLocalResource.java | 2 +- .../rtree/dataflow/LSMRTreeLocalResource.java | 4 +-- .../LSMRTreeWithAntiMatterLocalResource.java | 4 +-- .../am/lsm/rtree/impls/AbstractLSMRTree.java | 14 ++++---- .../am/lsm/rtree/impls/ExternalRTree.java | 13 +++---- .../storage/am/lsm/rtree/impls/LSMRTree.java | 16 +++++---- .../rtree/impls/LSMRTreeMemoryComponent.java | 8 ++--- .../impls/LSMRTreeWithAntiMatterTuples.java | 7 ++-- .../am/lsm/rtree/utils/LSMRTreeUtils.java | 27 ++++++++------- .../apache/hyracks/storage/common/IIndex.java | 7 ++-- .../am/lsm/btree/LSMBTreeBulkLoadTest.java | 4 +-- .../am/lsm/btree/LSMBTreeDeleteTest.java | 4 +-- .../am/lsm/btree/LSMBTreeExamplesTest.java | 2 +- .../am/lsm/btree/LSMBTreeFileManagerTest.java | 8 ++--- .../am/lsm/btree/LSMBTreeFilterMergeTest.java | 4 +-- .../btree/LSMBTreeFilterMergeTestDriver.java | 3 +- .../am/lsm/btree/LSMBTreeInsertTest.java | 4 +-- .../am/lsm/btree/LSMBTreeLifecycleTest.java | 4 +-- .../storage/am/lsm/btree/LSMBTreeMergeTest.java | 4 +-- .../am/lsm/btree/LSMBTreeMergeTestDriver.java | 3 +- ...MBTreeModificationOperationCallbackTest.java | 9 +++-- .../am/lsm/btree/LSMBTreeMultiBulkLoadTest.java | 4 +-- .../btree/LSMBTreeScanDiskComponentsTest.java | 15 ++++---- .../LSMBTreeSearchOperationCallbackTest.java | 2 +- ...TreeUpdateInPlaceScanDiskComponentsTest.java | 11 +++--- .../am/lsm/btree/LSMBTreeUpdateInPlaceTest.java | 9 +++-- .../am/lsm/btree/LSMBTreeUpdateTest.java | 4 +-- .../storage/am/lsm/btree/impl/TestLsmBtree.java | 9 ++--- .../btree/impl/TestLsmBtreeLocalResource.java | 4 +-- .../am/lsm/btree/impl/TestLsmBtreeUtil.java | 6 ++-- .../multithread/LSMBTreeMultiThreadTest.java | 4 +-- .../btree/multithread/LSMBTreeTestWorker.java | 5 +-- .../am/lsm/btree/perf/LSMTreeRunner.java | 6 ++-- .../am/lsm/btree/util/LSMBTreeTestContext.java | 10 +++--- .../am/lsm/btree/util/LSMBTreeTestHarness.java | 10 +++--- .../LSMInvertedIndexMergeTest.java | 3 +- .../PartitionedLSMInvertedIndexMergeTest.java | 3 +- .../common/LSMInvertedIndexTestHarness.java | 10 +++--- .../multithread/LSMInvertedIndexTestWorker.java | 4 +-- .../util/LSMInvertedIndexTestContext.java | 14 ++++---- .../am/lsm/rtree/LSMRTreeBulkLoadTest.java | 2 +- .../am/lsm/rtree/LSMRTreeDeleteTest.java | 2 +- .../am/lsm/rtree/LSMRTreeExamplesTest.java | 2 +- .../am/lsm/rtree/LSMRTreeInsertTest.java | 2 +- .../am/lsm/rtree/LSMRTreeLifecycleTest.java | 2 +- .../storage/am/lsm/rtree/LSMRTreeMergeTest.java | 2 +- .../am/lsm/rtree/LSMRTreeMergeTestDriver.java | 3 +- ...SMRTreeWithAntiMatterTuplesBulkLoadTest.java | 2 +- .../LSMRTreeWithAntiMatterTuplesDeleteTest.java | 2 +- ...SMRTreeWithAntiMatterTuplesExamplesTest.java | 2 +- .../LSMRTreeWithAntiMatterTuplesInsertTest.java | 2 +- ...MRTreeWithAntiMatterTuplesLifecycleTest.java | 2 +- .../LSMRTreeWithAntiMatterTuplesMergeTest.java | 2 +- .../multithread/LSMRTreeMultiThreadTest.java | 2 +- .../rtree/multithread/LSMRTreeTestWorker.java | 4 +-- ...TreeWithAntiMatterTuplesMultiThreadTest.java | 2 +- .../LSMRTreeWithAntiMatterTuplesTestWorker.java | 3 +- .../am/lsm/rtree/util/LSMRTreeTestContext.java | 6 ++-- .../am/lsm/rtree/util/LSMRTreeTestHarness.java | 10 +++--- ...LSMRTreeWithAntiMatterTuplesTestContext.java | 9 ++--- 106 files changed, 416 insertions(+), 368 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a0490ba6/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a0490ba6/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a0490ba6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a0490ba6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a0490ba6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a0490ba6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a0490ba6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a0490ba6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a0490ba6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ThreadCountingTracker.java ---------------------------------------------------------------------- diff --cc hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ThreadCountingTracker.java index 93ac669,6cb96aa..a7fbe06 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ThreadCountingTracker.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ThreadCountingTracker.java @@@ -57,8 -57,9 +57,8 @@@ public class ThreadCountingTracker impl // Flush will only be handled by last exiting thread. if (opType == LSMOperationType.MODIFICATION && threadRefCount.decrementAndGet() == 0 && index.hasFlushRequestForCurrentMutableComponent()) { - ILSMIndexAccessor accessor = - index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); + ILSMIndexAccessor accessor = index.createAccessor(NoOpIndexAccessParameters.INSTANCE); - accessor.scheduleFlush(NoOpIOOperationCallbackFactory.INSTANCE.createIoOpCallback()); + accessor.scheduleFlush(NoOpIOOperationCallbackFactory.INSTANCE.createIoOpCallback(index)); } } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a0490ba6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a0490ba6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a0490ba6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a0490ba6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a0490ba6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a0490ba6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a0490ba6/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndex.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a0490ba6/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a0490ba6/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java ---------------------------------------------------------------------- diff --cc hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java index 6574b6f,fdf027e..1af6779 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java @@@ -72,9 -68,9 +69,11 @@@ public class LSMBTreeModificationOperat @Override @Test public void modificationCallbackTest() throws Exception { - ILSMIndexAccessor accessor = (ILSMIndexAccessor) index.createAccessor(cb, NoOpOperationCallback.INSTANCE); + IndexAccessParameters actx = new IndexAccessParameters(cb, NoOpOperationCallback.INSTANCE); + ILSMIndexAccessor accessor = (ILSMIndexAccessor) index.createAccessor(actx); + + BlockingIOOperationCallbackWrapper ioOpCallback = + new BlockingIOOperationCallbackWrapper(((ILSMIndex) index).getIOOperationCallback()); for (int j = 0; j < 2; j++) { isFoundNull = true; for (int i = 0; i < NUM_TUPLES; i++) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a0490ba6/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a0490ba6/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceScanDiskComponentsTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a0490ba6/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a0490ba6/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a0490ba6/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/LSMInvertedIndexMergeTest.java ---------------------------------------------------------------------- diff --cc hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/LSMInvertedIndexMergeTest.java index ab6f3b9,ca91ef4..4657109 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/LSMInvertedIndexMergeTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/LSMInvertedIndexMergeTest.java @@@ -22,10 -22,9 +22,9 @@@ package org.apache.hyracks.storage.am.l import java.io.IOException; import org.apache.hyracks.storage.am.common.datagen.TupleGenerator; -import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback; +import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters; import org.apache.hyracks.storage.am.config.AccessMethodTestsConfig; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; - import org.apache.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallbackFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.common.AbstractInvertedIndexLoadTest; import org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndex; import org.apache.hyracks.storage.am.lsm.invertedindex.util.LSMInvertedIndexTestContext; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a0490ba6/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/PartitionedLSMInvertedIndexMergeTest.java ---------------------------------------------------------------------- diff --cc hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/PartitionedLSMInvertedIndexMergeTest.java index f57e797,f4ff9cd..584de0c --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/PartitionedLSMInvertedIndexMergeTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/PartitionedLSMInvertedIndexMergeTest.java @@@ -23,10 -23,9 +23,9 @@@ import java.io.IOException import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.storage.am.common.datagen.TupleGenerator; -import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback; +import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters; import org.apache.hyracks.storage.am.config.AccessMethodTestsConfig; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; - import org.apache.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallbackFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.common.AbstractInvertedIndexLoadTest; import org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndex; import org.apache.hyracks.storage.am.lsm.invertedindex.util.LSMInvertedIndexTestContext;
