abdullah alamoudi has uploaded a new change for review. https://asterix-gerrit.ics.uci.edu/2017
Change subject: Component Rollback Through flushes and merges ...................................................................... Component Rollback Through flushes and merges Change-Id: I178656207bfa1d15e6ae5ff2403a16df33940773 --- M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/ophelpers/IndexOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/LSMOperationType.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java D hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentState.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/RollbackDiskComponent.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java 24 files changed, 296 insertions(+), 92 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/17/2017/1 diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/ophelpers/IndexOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/ophelpers/IndexOperation.java index 71a3f71..b356ad6 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/ophelpers/IndexOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/ophelpers/IndexOperation.java @@ -33,5 +33,7 @@ FULL_MERGE, FLUSH, REPLICATE, - DISK_COMPONENT_SCAN + DISK_COMPONENT_SCAN, + ROLLBACK_MEMORY, + ROLLBACK_DISK } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java index 86d926f..77cc1da 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java @@ -376,7 +376,7 @@ // Not supported @Override - public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException { + public ILSMDiskComponent doFlush(ILSMIOOperation operation) throws HyracksDataException { throw new UnsupportedOperationException("flush not supported in LSM-Disk-Only-BTree"); } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java index 7462c7a..5f73e53 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java @@ -325,7 +325,7 @@ } @Override - public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException { + public ILSMDiskComponent doFlush(ILSMIOOperation operation) throws HyracksDataException { throw HyracksDataException.create(ErrorCode.FLUSH_NOT_SUPPORTED_IN_EXTERNAL_INDEX); } @@ -378,7 +378,7 @@ } @Override - public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException { + public ILSMDiskComponent doMerge(ILSMIOOperation operation) throws HyracksDataException { LSMBTreeWithBuddyMergeOperation mergeOp = (LSMBTreeWithBuddyMergeOperation) operation; IIndexCursor cursor = mergeOp.getCursor(); ISearchPredicate btreeSearchPred = new RangePredicate(null, null, true, true, null, null); @@ -816,8 +816,8 @@ @Override protected ILSMIOOperation createFlushOperation(AbstractLSMIndexOperationContext opCtx, - ILSMMemoryComponent flushingComponent, LSMComponentFileReferences componentFileRefs, - ILSMIOOperationCallback callback) throws HyracksDataException { + LSMComponentFileReferences componentFileRefs, ILSMIOOperationCallback callback) + throws HyracksDataException { return null; } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java index 24b408c..2fcb9a2 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java @@ -294,7 +294,7 @@ } @Override - public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException { + public ILSMDiskComponent doFlush(ILSMIOOperation operation) throws HyracksDataException { LSMBTreeFlushOperation flushOp = (LSMBTreeFlushOperation) operation; LSMBTreeMemoryComponent flushingComponent = (LSMBTreeMemoryComponent) flushOp.getFlushingComponent(); IIndexAccessor accessor = flushingComponent.getBTree().createAccessor(NoOpOperationCallback.INSTANCE, @@ -360,7 +360,7 @@ } @Override - public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException { + public ILSMDiskComponent doMerge(ILSMIOOperation operation) throws HyracksDataException { LSMBTreeMergeOperation mergeOp = (LSMBTreeMergeOperation) operation; IIndexCursor cursor = mergeOp.getCursor(); RangePredicate rangePred = new RangePredicate(null, null, true, true, null, null); @@ -475,10 +475,9 @@ @Override protected ILSMIOOperation createFlushOperation(AbstractLSMIndexOperationContext opCtx, - ILSMMemoryComponent flushingComponent, LSMComponentFileReferences componentFileRefs, - ILSMIOOperationCallback callback) { + LSMComponentFileReferences componentFileRefs, ILSMIOOperationCallback callback) { ILSMIndexAccessor accessor = createAccessor(opCtx); - return new LSMBTreeFlushOperation(accessor, flushingComponent, componentFileRefs.getInsertIndexFileReference(), + return new LSMBTreeFlushOperation(accessor, componentFileRefs.getInsertIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir().getAbsolutePath()); } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java index 4a06778..e3424e5 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java @@ -21,16 +21,14 @@ import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent; import org.apache.hyracks.storage.am.lsm.common.impls.FlushOperation; public class LSMBTreeFlushOperation extends FlushOperation { private final FileReference bloomFilterFlushTarget; - public LSMBTreeFlushOperation(ILSMIndexAccessor accessor, ILSMMemoryComponent flushingComponent, - FileReference flushTarget, FileReference bloomFilterFlushTarget, ILSMIOOperationCallback callback, - String indexIdentifier) { - super(accessor, flushingComponent, flushTarget, callback, indexIdentifier); + public LSMBTreeFlushOperation(ILSMIndexAccessor accessor, FileReference flushTarget, + FileReference bloomFilterFlushTarget, ILSMIOOperationCallback callback, String indexIdentifier) { + super(accessor, flushTarget, callback, indexIdentifier); this.bloomFilterFlushTarget = bloomFilterFlushTarget; } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java index aed641d..0632340 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java @@ -22,6 +22,7 @@ import org.apache.hyracks.storage.am.lsm.common.impls.DiskComponentMetadata; public interface ILSMDiskComponent extends ILSMComponent { + @Override default LSMComponentType getType() { return LSMComponentType.DISK; @@ -49,6 +50,7 @@ /** * Return the component Id of this disk component from its metadata + * * @return * @throws HyracksDataException */ diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java index c0a3f2d..e7e21fb 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java @@ -19,6 +19,7 @@ package org.apache.hyracks.storage.am.lsm.common.api; import java.util.List; +import java.util.function.Predicate; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.data.std.api.IValueReference; @@ -242,4 +243,13 @@ */ void batchOperate(ILSMIndexOperationContext ctx, FrameTupleAccessor accessor, FrameTupleReference tuple, IFrameTupleProcessor processor, IFrameOperationCallback frameOpCallback) throws HyracksDataException; + + /** + * Rollback components that match the passed predicate + * + * @param ctx + * @param predicate + * @throws HyracksDataException + */ + void rollback(ILSMIndexOperationContext ctx, Predicate<ILSMComponent> predicate) throws HyracksDataException; } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java index 79360d5..0d16fbf 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java @@ -55,4 +55,9 @@ @Override Boolean call() throws HyracksDataException; + + /** + * @return the accessor of the operation + */ + ILSMIndexAccessor getAccessor(); } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java index 1042df2..3d1da46 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java @@ -19,6 +19,7 @@ package org.apache.hyracks.storage.am.lsm.common.api; import java.util.List; +import java.util.function.Predicate; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.data.std.api.IValueReference; @@ -34,6 +35,11 @@ * concurrent operations). */ public interface ILSMIndexAccessor extends IIndexAccessor { + + /** + * @return the operation context associated with the accessor + */ + ILSMIndexOperationContext getOpContext(); /** * Schedule a flush operation @@ -245,4 +251,12 @@ * If the BufferCache throws while un/pinning or un/latching. */ void scanDiskComponents(IIndexCursor cursor) throws HyracksDataException; + + /** + * Rollback components that match the passed predicate + * + * @param filter + * @throws HyracksDataException + */ + void rollback(Predicate<ILSMComponent> predicate) throws HyracksDataException; } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/LSMOperationType.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/LSMOperationType.java index 63d2697..188a37b 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/LSMOperationType.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/LSMOperationType.java @@ -25,5 +25,6 @@ FLUSH, MERGE, REPLICATE, - DISK_COMPONENT_SCAN + DISK_COMPONENT_SCAN, + ROLLBACK } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java index 09e236e..b00c3f5 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java @@ -53,6 +53,7 @@ return target; } + @Override public ILSMIndexAccessor getAccessor() { return accessor; } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java index c5bf2ca..3da5b26 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java @@ -196,6 +196,8 @@ } // What if more than one memory component needs flushing?? + // Can this ever be the case?? We don't switch memory components until flush has been scheduled for the + // current one. protected void flushMemoryComponents() throws HyracksDataException { BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback); ILSMIndexAccessor accessor = createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); @@ -275,6 +277,7 @@ case UPDATE: case PHYSICALDELETE: case FLUSH: + case ROLLBACK_MEMORY: case DELETE: case UPSERT: operationalComponents.add(memoryComponents.get(cmc)); @@ -302,6 +305,7 @@ break; case MERGE: + case ROLLBACK_DISK: operationalComponents.addAll(ctx.getComponentsToBeMerged()); break; case FULL_MERGE: @@ -326,13 +330,12 @@ @Override public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback) throws HyracksDataException { - ILSMMemoryComponent flushingComponent = (ILSMMemoryComponent) ctx.getComponentHolder().get(0); LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference(); AbstractLSMIndexOperationContext opCtx = createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); - opCtx.setOperation(IndexOperation.FLUSH); - opCtx.getComponentHolder().add(flushingComponent); - ILSMIOOperation flushOp = createFlushOperation(opCtx, flushingComponent, componentFileRefs, callback); + opCtx.setOperation(ctx.getOperation()); + opCtx.getComponentHolder().addAll(ctx.getComponentHolder()); + ILSMIOOperation flushOp = createFlushOperation(opCtx, componentFileRefs, callback); ioScheduler.scheduleOperation(flushOp); } @@ -409,7 +412,9 @@ @Override public void addDiskComponent(ILSMDiskComponent c) throws HyracksDataException { - diskComponents.add(0, c); + if (c != RollbackDiskComponent.INSTANCE) { + diskComponents.add(0, c); + } } @Override @@ -417,7 +422,9 @@ throws HyracksDataException { int swapIndex = diskComponents.indexOf(mergedComponents.get(0)); diskComponents.removeAll(mergedComponents); - diskComponents.add(swapIndex, newComponent); + if (newComponent != RollbackDiskComponent.INSTANCE) { + diskComponents.add(swapIndex, newComponent); + } } @Override @@ -623,6 +630,21 @@ return size; } + @Override + public final ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException { + ILSMIndexAccessor accessor = operation.getAccessor(); + ILSMIndexOperationContext opCtx = accessor.getOpContext(); + return opCtx.getOperation() == IndexOperation.ROLLBACK_MEMORY ? RollbackDiskComponent.INSTANCE + : doFlush(operation); + } + + @Override + public final ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException { + ILSMIndexAccessor accessor = operation.getAccessor(); + ILSMIndexOperationContext opCtx = accessor.getOpContext(); + return opCtx.getOperation() == IndexOperation.ROLLBACK_DISK ? RollbackDiskComponent.INSTANCE : doMerge(operation); + } + public abstract Set<String> getLSMComponentPhysicalFiles(ILSMComponent newComponent); protected abstract void allocateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException; @@ -656,11 +678,14 @@ throws HyracksDataException; protected abstract ILSMIOOperation createFlushOperation(AbstractLSMIndexOperationContext opCtx, - ILSMMemoryComponent flushingComponent, LSMComponentFileReferences componentFileRefs, - ILSMIOOperationCallback callback) throws HyracksDataException; + LSMComponentFileReferences componentFileRefs, ILSMIOOperationCallback callback) throws HyracksDataException; protected abstract ILSMIOOperation createMergeOperation(AbstractLSMIndexOperationContext opCtx, List<ILSMComponent> mergingComponents, LSMComponentFileReferences mergeFileRefs, ILSMIOOperationCallback callback) throws HyracksDataException; + protected abstract ILSMDiskComponent doFlush(ILSMIOOperation operation) throws HyracksDataException; + + protected abstract ILSMDiskComponent doMerge(ILSMIOOperation operation) throws HyracksDataException; + } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java index cafb008..3579b12 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java @@ -25,16 +25,12 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent; public class FlushOperation extends AbstractIoOperation implements Comparable<FlushOperation> { - protected final ILSMMemoryComponent flushingComponent; - - public FlushOperation(ILSMIndexAccessor accessor, ILSMMemoryComponent flushingComponent, FileReference target, - ILSMIOOperationCallback callback, String indexIdentifier) { + public FlushOperation(ILSMIndexAccessor accessor, FileReference target, ILSMIOOperationCallback callback, + String indexIdentifier) { super(accessor, target, callback, indexIdentifier); - this.flushingComponent = flushingComponent; } @Override @@ -59,7 +55,7 @@ } public ILSMComponent getFlushingComponent() { - return flushingComponent; + return accessor.getOpContext().getComponentHolder().get(0); } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentState.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentState.java deleted file mode 100644 index 094b6c6..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentState.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hyracks.storage.am.lsm.common.impls; - -public enum LSMComponentState { - FLUSHING, - MERGING, - DONE_FLUSHING, - DONE_MERGING -} diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java index 35c93ba..864b6a8 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java @@ -23,6 +23,7 @@ import java.util.LinkedList; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; @@ -217,7 +218,6 @@ try { synchronized (opTracker) { try { - /** * [flow control] * If merge operations are lagged according to the merge policy, @@ -747,6 +747,89 @@ } @Override + public void rollback(ILSMIndexOperationContext ctx, Predicate<ILSMComponent> predicate) + throws HyracksDataException { + synchronized (opTracker) { + while (true) { + boolean canRollback = true; + // ensure that no flushes are taking place + for (ILSMMemoryComponent memComponent : lsmIndex.getMemoryComponents()) { + if (memComponent.getState() == ComponentState.READABLE_UNWRITABLE_FLUSHING) { + canRollback = false; + break; + } + } + if (canRollback) { + // ensure that no merges are taking place + for (ILSMDiskComponent memComponent : lsmIndex.getImmutableComponents()) { + if (memComponent.getState() == ComponentState.READABLE_MERGING) { + canRollback = false; + break; + } + } + } + if (canRollback) { + break; + } + try { + opTracker.wait(); + } catch (InterruptedException e) { + LOGGER.log(Level.WARNING, "Interrupted while attempting component level rollback", e); + Thread.currentThread().interrupt(); + throw HyracksDataException.create(e); + } + } + // We always start with the memory component + ILSMMemoryComponent memComponent = lsmIndex.getCurrentMemoryComponent(); + if (!predicate.test(memComponent)) { + // It is a no op since even memory component is not to be rolled back + return; + } + } + // Here, we are releasing the opTracker to allow other operations: + // (searches, rollback flush we will call, rollback merge we will call). + // This will break if: + // - we get a flush call (can we use a flag to prevent all other flushes?) + // - we get a full merge (can we use a flag to prevent all other merges?) + // + // schedule a rollback fo flushed component + ctx.reset(); + BlockingIOOperationCallbackWrapper ioCallback = + new BlockingIOOperationCallbackWrapper(lsmIndex.getIOOperationCallback()); + ctx.setOperation(IndexOperation.ROLLBACK_MEMORY); + // ScheduleFlush is actually a try operation + scheduleFlush(ctx, ioCallback); + // Note that the following call assumes that the flush has succeeded. + try { + ioCallback.waitForIO(); + } catch (InterruptedException e) { + LOGGER.log(Level.WARNING, "Rollback has been interrupted. returning"); + Thread.currentThread().interrupt(); + throw HyracksDataException.create(e); + } + ctx.reset(); + ioCallback = new BlockingIOOperationCallbackWrapper(lsmIndex.getIOOperationCallback()); + ctx.setOperation(IndexOperation.ROLLBACK_DISK); + List<ILSMDiskComponent> diskComponents = lsmIndex.getImmutableComponents(); + for (ILSMDiskComponent component : diskComponents) { + if (!predicate.test(component)) { + break; + } + ctx.getComponentsToBeMerged().add(component); + } + // ScheduleMerge is actually a try operation + scheduleMerge(ctx, ioCallback); + // Note that the following call assumes that the merge has succeeded. + try { + ioCallback.waitForIO(); + } catch (InterruptedException e) { + LOGGER.log(Level.WARNING, "Rollback has been interrupted. returning"); + Thread.currentThread().interrupt(); + throw HyracksDataException.create(e); + } + } + + @Override public String toString() { return getClass().getSimpleName() + ":" + lsmIndex; } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java index a45225d..9f65903 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java @@ -20,6 +20,7 @@ package org.apache.hyracks.storage.am.lsm.common.impls; import java.util.List; +import java.util.function.Predicate; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.data.std.api.IValueReference; @@ -30,6 +31,7 @@ import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; import org.apache.hyracks.storage.am.lsm.common.api.IFrameOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.IFrameTupleProcessor; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation; @@ -224,4 +226,14 @@ public String toString() { return getClass().getSimpleName() + ':' + lsmHarness.toString(); } + + @Override + public void rollback(Predicate<ILSMComponent> predicate) throws HyracksDataException { + lsmHarness.rollback(ctx, predicate); + } + + @Override + public ILSMIndexOperationContext getOpContext() { + return ctx; + } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/RollbackDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/RollbackDiskComponent.java new file mode 100644 index 0000000..c0b7d3b --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/RollbackDiskComponent.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.storage.am.lsm.common.impls; + +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentId; +import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType; + +public class RollbackDiskComponent implements ILSMDiskComponent { + public static final RollbackDiskComponent INSTANCE = new RollbackDiskComponent(); + + private RollbackDiskComponent() { + } + + @Override + public boolean threadEnter(LSMOperationType opType, boolean isMutableComponent) throws HyracksDataException { + return true; + } + + @Override + public void threadExit(LSMOperationType opType, boolean failedOperation, boolean isMutableComponent) + throws HyracksDataException { + } + + @Override + public ComponentState getState() { + return ComponentState.INACTIVE; + } + + @Override + public ILSMComponentFilter getLSMComponentFilter() { + return null; + } + + @Override + public DiskComponentMetadata getMetadata() { + return null; + } + + @Override + public long getComponentSize() { + return 0; + } + + @Override + public int getFileReferenceCount() { + return 0; + } + + @Override + public void destroy() throws HyracksDataException { + } + + @Override + public ILSMDiskComponentId getComponentId() throws HyracksDataException { + return null; + } + +} diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java index 1f2a810..2526f37 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java @@ -218,7 +218,7 @@ if (ctx.getIndexTuple() != null) { ctx.getIndexTuple().reset(tuple); indexTuple = ctx.getIndexTuple(); - ((InMemoryInvertedIndexAccessor)(ctx.getCurrentMutableInvIndexAccessors())).resetLogTuple(tuple); + ((InMemoryInvertedIndexAccessor) (ctx.getCurrentMutableInvIndexAccessors())).resetLogTuple(tuple); } else { indexTuple = tuple; } @@ -329,7 +329,7 @@ } @Override - public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException { + public ILSMDiskComponent doFlush(ILSMIOOperation operation) throws HyracksDataException { LSMInvertedIndexFlushOperation flushOp = (LSMInvertedIndexFlushOperation) operation; // Create an inverted index instance to be bulk loaded. @@ -407,7 +407,7 @@ } @Override - public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException { + public ILSMDiskComponent doMerge(ILSMIOOperation operation) throws HyracksDataException { LSMInvertedIndexMergeOperation mergeOp = (LSMInvertedIndexMergeOperation) operation; IIndexCursor cursor = mergeOp.getCursor(); @@ -708,12 +708,11 @@ @Override protected ILSMIOOperation createFlushOperation(AbstractLSMIndexOperationContext opCtx, - ILSMMemoryComponent flushingComponent, LSMComponentFileReferences componentFileRefs, - ILSMIOOperationCallback callback) throws HyracksDataException { + LSMComponentFileReferences componentFileRefs, ILSMIOOperationCallback callback) + throws HyracksDataException { return new LSMInvertedIndexFlushOperation(new LSMInvertedIndexAccessor(getLsmHarness(), opCtx), - flushingComponent, componentFileRefs.getInsertIndexFileReference(), - componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), - callback, fileManager.getBaseDir().getAbsolutePath()); + componentFileRefs.getInsertIndexFileReference(), componentFileRefs.getDeleteIndexFileReference(), + componentFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir().getAbsolutePath()); } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java index dddd14a..66893ae 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java @@ -19,12 +19,14 @@ package org.apache.hyracks.storage.am.lsm.invertedindex.impls; import java.util.List; +import java.util.function.Predicate; import org.apache.hyracks.api.exceptions.ErrorCode; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.data.std.api.IValueReference; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation; @@ -213,4 +215,14 @@ public String toString() { return getClass().getSimpleName() + ':' + lsmHarness.toString(); } + + @Override + public void rollback(Predicate<ILSMComponent> predicate) throws HyracksDataException { + lsmHarness.rollback(ctx, predicate); + } + + @Override + public ILSMIndexOperationContext getOpContext() { + return ctx; + } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java index df4f095..2106f6a 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java @@ -22,17 +22,16 @@ import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent; import org.apache.hyracks.storage.am.lsm.common.impls.FlushOperation; public class LSMInvertedIndexFlushOperation extends FlushOperation { private final FileReference deletedKeysBTreeFlushTarget; private final FileReference bloomFilterFlushTarget; - public LSMInvertedIndexFlushOperation(ILSMIndexAccessor accessor, ILSMMemoryComponent flushingComponent, - FileReference flushTarget, FileReference deletedKeysBTreeFlushTarget, FileReference bloomFilterFlushTarget, + public LSMInvertedIndexFlushOperation(ILSMIndexAccessor accessor, FileReference flushTarget, + FileReference deletedKeysBTreeFlushTarget, FileReference bloomFilterFlushTarget, ILSMIOOperationCallback callback, String indexIdentifier) { - super(accessor, flushingComponent, flushTarget, callback, indexIdentifier); + super(accessor, flushTarget, callback, indexIdentifier); this.deletedKeysBTreeFlushTarget = deletedKeysBTreeFlushTarget; this.bloomFilterFlushTarget = bloomFilterFlushTarget; } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java index e39c3f9..1380e6e 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java @@ -258,7 +258,7 @@ // This can be done in a better way by creating a method boolean // keepDeletedTuples(mergedComponents); @Override - public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException { + public ILSMDiskComponent doMerge(ILSMIOOperation operation) throws HyracksDataException { LSMRTreeMergeOperation mergeOp = (LSMRTreeMergeOperation) operation; IIndexCursor cursor = mergeOp.getCursor(); ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null); @@ -424,7 +424,7 @@ // Not supported @Override - public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException { + public ILSMDiskComponent doFlush(ILSMIOOperation operation) throws HyracksDataException { throw new UnsupportedOperationException("flush not supported in LSM-Disk-Only-RTree"); } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java index 0a47aea..8eed39a 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java @@ -56,7 +56,6 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy; import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker; import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache; @@ -163,7 +162,7 @@ } @Override - public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException { + public ILSMDiskComponent doFlush(ILSMIOOperation operation) throws HyracksDataException { LSMRTreeFlushOperation flushOp = (LSMRTreeFlushOperation) operation; LSMRTreeMemoryComponent flushingComponent = (LSMRTreeMemoryComponent) flushOp.getFlushingComponent(); // Renaming order is critical because we use assume ordering when we @@ -263,7 +262,7 @@ } @Override - public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException { + public ILSMDiskComponent doMerge(ILSMIOOperation operation) throws HyracksDataException { LSMRTreeMergeOperation mergeOp = (LSMRTreeMergeOperation) operation; IIndexCursor cursor = mergeOp.getCursor(); ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null); @@ -428,10 +427,10 @@ @Override protected ILSMIOOperation createFlushOperation(AbstractLSMIndexOperationContext opCtx, - ILSMMemoryComponent flushingComponent, LSMComponentFileReferences componentFileRefs, - ILSMIOOperationCallback callback) throws HyracksDataException { + LSMComponentFileReferences componentFileRefs, ILSMIOOperationCallback callback) + throws HyracksDataException { LSMRTreeAccessor accessor = new LSMRTreeAccessor(getLsmHarness(), opCtx, buddyBTreeFields); - return new LSMRTreeFlushOperation(accessor, flushingComponent, componentFileRefs.getInsertIndexFileReference(), + return new LSMRTreeFlushOperation(accessor, componentFileRefs.getInsertIndexFileReference(), componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir().getAbsolutePath()); } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java index f3e45ac..6991c56 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java @@ -21,7 +21,6 @@ import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent; import org.apache.hyracks.storage.am.lsm.common.impls.FlushOperation; public class LSMRTreeFlushOperation extends FlushOperation { @@ -29,10 +28,9 @@ private final FileReference btreeFlushTarget; private final FileReference bloomFilterFlushTarget; - public LSMRTreeFlushOperation(ILSMIndexAccessor accessor, ILSMMemoryComponent flushingComponent, - FileReference flushTarget, FileReference btreeFlushTarget, FileReference bloomFilterFlushTarget, - ILSMIOOperationCallback callback, String indexIdentifier) { - super(accessor, flushingComponent, flushTarget, callback, indexIdentifier); + public LSMRTreeFlushOperation(ILSMIndexAccessor accessor, FileReference flushTarget, FileReference btreeFlushTarget, + FileReference bloomFilterFlushTarget, ILSMIOOperationCallback callback, String indexIdentifier) { + super(accessor, flushTarget, callback, indexIdentifier); this.btreeFlushTarget = btreeFlushTarget; this.bloomFilterFlushTarget = bloomFilterFlushTarget; } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java index 94648fb..52729e5 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java @@ -47,7 +47,6 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy; import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker; import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache; @@ -124,7 +123,7 @@ } @Override - public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException { + public ILSMDiskComponent doFlush(ILSMIOOperation operation) throws HyracksDataException { LSMRTreeFlushOperation flushOp = (LSMRTreeFlushOperation) operation; // Renaming order is critical because we use assume ordering when we // read the file names when we open the tree. @@ -212,7 +211,7 @@ } @Override - public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException { + public ILSMDiskComponent doMerge(ILSMIOOperation operation) throws HyracksDataException { MergeOperation mergeOp = (MergeOperation) operation; IIndexCursor cursor = mergeOp.getCursor(); ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null); @@ -336,11 +335,11 @@ @Override protected ILSMIOOperation createFlushOperation(AbstractLSMIndexOperationContext opCtx, - ILSMMemoryComponent flushingComponent, LSMComponentFileReferences componentFileRefs, - ILSMIOOperationCallback callback) throws HyracksDataException { + LSMComponentFileReferences componentFileRefs, ILSMIOOperationCallback callback) + throws HyracksDataException { ILSMIndexAccessor accessor = new LSMTreeIndexAccessor(getLsmHarness(), opCtx, cursorFactory); - return new LSMRTreeFlushOperation(accessor, flushingComponent, componentFileRefs.getInsertIndexFileReference(), - null, null, callback, fileManager.getBaseDir().getAbsolutePath()); + return new LSMRTreeFlushOperation(accessor, componentFileRefs.getInsertIndexFileReference(), null, null, + callback, fileManager.getBaseDir().getAbsolutePath()); } @Override -- To view, visit https://asterix-gerrit.ics.uci.edu/2017 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I178656207bfa1d15e6ae5ff2403a16df33940773 Gerrit-PatchSet: 1 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: abdullah alamoudi <bamou...@gmail.com>