http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java index 7bf5322..b2660a4 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java @@ -81,7 +81,11 @@ public class InMemoryInvertedListCursor implements IInvertedListCursor { @Override public int compareTo(IInvertedListCursor cursor) { - return size() - cursor.size(); + try { + return size() - cursor.size(); + } catch (HyracksDataException hde) { + throw new IllegalStateException(hde); + } } public void reset(ITupleReference tuple) throws HyracksDataException { @@ -113,7 +117,7 @@ public class InMemoryInvertedListCursor implements IInvertedListCursor { @Override public void unpinPages() throws HyracksDataException { if (cursorNeedsClose) { - btreeCursor.destroy(); + btreeCursor.close(); cursorNeedsClose = false; } } @@ -135,29 +139,22 @@ public class InMemoryInvertedListCursor implements IInvertedListCursor { } @Override - public int size() { + public int size() throws HyracksDataException { if (numElements < 0) { btreePred.setLowKeyComparator(tokenFieldsCmp); btreePred.setHighKeyComparator(tokenFieldsCmp); btreePred.setLowKey(tokenTuple, true); btreePred.setHighKey(tokenTuple, true); - // Perform the count. + btreeAccessor.search(countingCursor, btreePred); try { - btreeAccessor.search(countingCursor, btreePred); while (countingCursor.hasNext()) { countingCursor.next(); ITupleReference countTuple = countingCursor.getTuple(); numElements = IntegerPointable.getInteger(countTuple.getFieldData(0), countTuple.getFieldStart(0)); } - } catch (HyracksDataException e) { - e.printStackTrace(); } finally { - try { - countingCursor.destroy(); - } catch (HyracksDataException e) { - e.printStackTrace(); - } + countingCursor.close(); } } return numElements; @@ -197,7 +194,6 @@ public class InMemoryInvertedListCursor implements IInvertedListCursor { try { containsKey = btreeCursor.hasNext(); } finally { - btreeCursor.destroy(); btreeCursor.close(); btreeSearchTuple.removeLastTuple(); } @@ -219,7 +215,6 @@ public class InMemoryInvertedListCursor implements IInvertedListCursor { strBuilder.append(o.toString() + " "); } } finally { - btreeCursor.destroy(); btreeCursor.close(); } btreeAccessor.search(btreeCursor, btreePred);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java index 75bdbc4..eec2993 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java @@ -70,10 +70,7 @@ public class FixedSizeElementInvertedListCursor implements IInvertedListCursor { @Override public boolean hasNext() { - if (currentElementIx < numElements) - return true; - else - return false; + return currentElementIx < numElements; } @Override @@ -203,8 +200,9 @@ public class FixedSizeElementInvertedListCursor implements IInvertedListCursor { DataInput dataIn = new DataInputStream(inStream); Object o = serdes[i].deserialize(dataIn); strBuilder.append(o.toString()); - if (i + 1 < tuple.getFieldCount()) + if (i + 1 < tuple.getFieldCount()) { strBuilder.append(","); + } } strBuilder.append(" "); } @@ -217,6 +215,7 @@ public class FixedSizeElementInvertedListCursor implements IInvertedListCursor { return strBuilder.toString(); } + @Override @SuppressWarnings("rawtypes") public String printCurrentElement(ISerializerDeserializer[] serdes) throws HyracksDataException { StringBuilder strBuilder = new StringBuilder(); @@ -226,8 +225,9 @@ public class FixedSizeElementInvertedListCursor implements IInvertedListCursor { DataInput dataIn = new DataInputStream(inStream); Object o = serdes[i].deserialize(dataIn); strBuilder.append(o.toString()); - if (i + 1 < tuple.getFieldCount()) + if (i + 1 < tuple.getFieldCount()) { strBuilder.append(","); + } } return strBuilder.toString(); } @@ -249,17 +249,23 @@ public class FixedSizeElementInvertedListCursor implements IInvertedListCursor { } } - if (begin > arr.length - 1) + if (begin > arr.length - 1) { return -1; - if (key < arr[begin]) + } + if (key < arr[begin]) { return begin; - else + } else { return -1; + } } @Override public int compareTo(IInvertedListCursor invListCursor) { - return numElements - invListCursor.size(); + try { + return numElements - invListCursor.size(); + } catch (HyracksDataException hde) { + throw new IllegalStateException(hde); + } } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java index 4942eda..8e8cb13 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java @@ -206,7 +206,6 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex { listCursor.reset(0, 0, 0, 0); } } finally { - ctx.getBtreeCursor().destroy(); ctx.getBtreeCursor().close(); } } @@ -420,6 +419,7 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex { private final OnDiskInvertedIndex index; private final IInvertedIndexSearcher searcher; private final IIndexOperationContext opCtx = new OnDiskInvertedIndexOpContext(btree); + private boolean destroyed = false; public OnDiskInvertedIndexAccessor(OnDiskInvertedIndex index) throws HyracksDataException { this.index = index; @@ -483,6 +483,15 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex { public void upsert(ITupleReference tuple) throws HyracksDataException { throw new UnsupportedOperationException("Upsert not supported by inverted index."); } + + @Override + public void destroy() throws HyracksDataException { + if (destroyed) { + return; + } + destroyed = true; + opCtx.destroy(); + } } @Override @@ -542,24 +551,38 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex { btree.validate(); // Scan the btree and validate the order of elements in each inverted-list. IIndexAccessor btreeAccessor = btree.createAccessor(NoOpIndexAccessParameters.INSTANCE); - IIndexCursor btreeCursor = btreeAccessor.createSearchCursor(false); - MultiComparator btreeCmp = MultiComparator.create(btree.getComparatorFactories()); - RangePredicate rangePred = new RangePredicate(null, null, true, true, btreeCmp, btreeCmp); + try { + MultiComparator btreeCmp = MultiComparator.create(btree.getComparatorFactories()); + RangePredicate rangePred = new RangePredicate(null, null, true, true, btreeCmp, btreeCmp); + IIndexCursor btreeCursor = btreeAccessor.createSearchCursor(false); + try { + btreeAccessor.search(btreeCursor, rangePred); + try { + doValidate(btreeCursor); + } finally { + btreeCursor.close(); + } + } finally { + btreeCursor.destroy(); + } + } finally { + btreeAccessor.destroy(); + } + } + + private void doValidate(IIndexCursor btreeCursor) throws HyracksDataException { int[] fieldPermutation = new int[tokenTypeTraits.length]; for (int i = 0; i < tokenTypeTraits.length; i++) { fieldPermutation[i] = i; } PermutingTupleReference tokenTuple = new PermutingTupleReference(fieldPermutation); - + // Search key for finding an inverted-list in the actual index. + ArrayTupleBuilder prevBuilder = new ArrayTupleBuilder(invListTypeTraits.length); + ArrayTupleReference prevTuple = new ArrayTupleReference(); IInvertedIndexAccessor invIndexAccessor = createAccessor(NoOpIndexAccessParameters.INSTANCE); - IInvertedListCursor invListCursor = invIndexAccessor.createInvertedListCursor(); - MultiComparator invListCmp = MultiComparator.create(invListCmpFactories); - try { - // Search key for finding an inverted-list in the actual index. - ArrayTupleBuilder prevBuilder = new ArrayTupleBuilder(invListTypeTraits.length); - ArrayTupleReference prevTuple = new ArrayTupleReference(); - btreeAccessor.search(btreeCursor, rangePred); + IInvertedListCursor invListCursor = invIndexAccessor.createInvertedListCursor(); + MultiComparator invListCmp = MultiComparator.create(invListCmpFactories); while (btreeCursor.hasNext()) { btreeCursor.next(); tokenTuple.reset(btreeCursor.getTuple()); @@ -578,9 +601,7 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex { invListCursor.next(); ITupleReference invListElement = invListCursor.getTuple(); // Compare with previous element. - if (invListCmp.compare(invListElement, prevTuple) <= 0) { - throw new HyracksDataException("Index validation failed."); - } + validateWithPrevious(invListCmp, invListElement, prevTuple); // Set new prevTuple. TupleUtils.copyTuple(prevBuilder, invListElement, invListElement.getFieldCount()); prevTuple.reset(prevBuilder.getFieldEndOffsets(), prevBuilder.getByteArray()); @@ -590,7 +611,14 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex { } } } finally { - btreeCursor.destroy(); + invIndexAccessor.destroy(); + } + } + + private void validateWithPrevious(MultiComparator invListCmp, ITupleReference invListElement, + ArrayTupleReference prevTuple) throws HyracksDataException { + if (invListCmp.compare(invListElement, prevTuple) <= 0) { + throw new HyracksDataException("Index validation failed."); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java index 623fc74..89d4e9a 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java @@ -19,6 +19,7 @@ package org.apache.hyracks.storage.am.lsm.invertedindex.ondisk; +import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.storage.am.btree.impls.BTree; import org.apache.hyracks.storage.am.btree.impls.RangePredicate; import org.apache.hyracks.storage.am.common.api.IIndexOperationContext; @@ -36,6 +37,7 @@ public class OnDiskInvertedIndexOpContext implements IIndexOperationContext { private MultiComparator searchCmp; // For prefix search on partitioned indexes. private MultiComparator prefixSearchCmp; + private boolean destroyed = false; public OnDiskInvertedIndexOpContext(BTree btree) { // TODO: Ignore opcallbacks for now. @@ -82,4 +84,16 @@ public class OnDiskInvertedIndexOpContext implements IIndexOperationContext { return prefixSearchCmp; } + @Override + public void destroy() throws HyracksDataException { + if (destroyed) { + return; + } + destroyed = true; + try { + btreeAccessor.destroy(); + } finally { + btreeCursor.destroy(); + } + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java index d1e6531..7af35ff 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java @@ -28,6 +28,7 @@ import org.apache.hyracks.storage.am.common.tuples.ConcatenatingTupleReference; import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInPlaceInvertedIndex; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor; +import org.apache.hyracks.storage.common.EnforcedIndexCursor; import org.apache.hyracks.storage.common.ICursorInitialState; import org.apache.hyracks.storage.common.IIndexAccessor; import org.apache.hyracks.storage.common.IIndexCursor; @@ -36,7 +37,7 @@ import org.apache.hyracks.storage.common.ISearchPredicate; /** * Scans a range of tokens, returning tuples containing a token and an inverted-list element. */ -public class OnDiskInvertedIndexRangeSearchCursor implements IIndexCursor { +public class OnDiskInvertedIndexRangeSearchCursor extends EnforcedIndexCursor { private final BTree btree; private final IIndexAccessor btreeAccessor; @@ -69,7 +70,7 @@ public class OnDiskInvertedIndexRangeSearchCursor implements IIndexCursor { } @Override - public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException { + public void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException { this.btreePred = (RangePredicate) searchPred; btreeAccessor.search(btreeCursor, btreePred); invListCursor.pinPages(); @@ -77,7 +78,7 @@ public class OnDiskInvertedIndexRangeSearchCursor implements IIndexCursor { } @Override - public boolean hasNext() throws HyracksDataException { + public boolean doHasNext() throws HyracksDataException { if (invListCursor.hasNext()) { return true; } @@ -100,7 +101,7 @@ public class OnDiskInvertedIndexRangeSearchCursor implements IIndexCursor { } @Override - public void next() throws HyracksDataException { + public void doNext() throws HyracksDataException { invListCursor.next(); if (concatTuple.hasMaxTuples()) { concatTuple.removeLastTuple(); @@ -109,7 +110,7 @@ public class OnDiskInvertedIndexRangeSearchCursor implements IIndexCursor { } @Override - public void destroy() throws HyracksDataException { + public void doDestroy() throws HyracksDataException { if (unpinNeeded) { invListCursor.unpinPages(); unpinNeeded = false; @@ -118,16 +119,16 @@ public class OnDiskInvertedIndexRangeSearchCursor implements IIndexCursor { } @Override - public void close() throws HyracksDataException { + public void doClose() throws HyracksDataException { if (unpinNeeded) { invListCursor.unpinPages(); unpinNeeded = false; } - btreeCursor.destroy(); + btreeCursor.close(); } @Override - public ITupleReference getTuple() { + public ITupleReference doGetTuple() { return concatTuple; } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexSearchCursor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexSearchCursor.java index 97f1f76..0563ec9 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexSearchCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexSearchCursor.java @@ -27,11 +27,11 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearcher; +import org.apache.hyracks.storage.common.EnforcedIndexCursor; import org.apache.hyracks.storage.common.ICursorInitialState; -import org.apache.hyracks.storage.common.IIndexCursor; import org.apache.hyracks.storage.common.ISearchPredicate; -public class OnDiskInvertedIndexSearchCursor implements IIndexCursor { +public class OnDiskInvertedIndexSearchCursor extends EnforcedIndexCursor { private List<ByteBuffer> resultBuffers; private int numResultBuffers; @@ -55,7 +55,7 @@ public class OnDiskInvertedIndexSearchCursor implements IIndexCursor { } @Override - public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException { + public void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException { currentBufferIndex = 0; tupleIndex = 0; resultBuffers = invIndexSearcher.getResultBuffers(); @@ -66,7 +66,7 @@ public class OnDiskInvertedIndexSearchCursor implements IIndexCursor { } @Override - public boolean hasNext() { + public boolean doHasNext() { if (currentBufferIndex < numResultBuffers && tupleIndex < fta.getTupleCount()) { return true; } else { @@ -75,7 +75,7 @@ public class OnDiskInvertedIndexSearchCursor implements IIndexCursor { } @Override - public void next() { + public void doNext() { frameTuple.reset(fta.getBuffer().array(), fta.getTupleStartOffset(tupleIndex)); resultTuple.reset(frameTuple); tupleIndex++; @@ -89,12 +89,12 @@ public class OnDiskInvertedIndexSearchCursor implements IIndexCursor { } @Override - public ITupleReference getTuple() { + public ITupleReference doGetTuple() { return resultTuple; } @Override - public void close() { + public void doClose() { currentBufferIndex = 0; tupleIndex = 0; invIndexSearcher.reset(); @@ -103,7 +103,7 @@ public class OnDiskInvertedIndexSearchCursor implements IIndexCursor { } @Override - public void destroy() throws HyracksDataException { + public void doDestroy() throws HyracksDataException { currentBufferIndex = 0; tupleIndex = 0; resultBuffers = null; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java index 01e0684..064a26d 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java @@ -87,8 +87,8 @@ public class PartitionedOnDiskInvertedIndex extends OnDiskInvertedIndex implemen } ctx.getBtreePred().setLowKey(lowSearchKey, true); ctx.getBtreePred().setHighKey(highSearchKey, true); - ctx.getBtreeAccessor().search(ctx.getBtreeCursor(), ctx.getBtreePred()); boolean tokenExists = false; + ctx.getBtreeAccessor().search(ctx.getBtreeCursor(), ctx.getBtreePred()); try { while (ctx.getBtreeCursor().hasNext()) { ctx.getBtreeCursor().next(); @@ -102,7 +102,6 @@ public class PartitionedOnDiskInvertedIndex extends OnDiskInvertedIndex implemen tokenExists = true; } } finally { - ctx.getBtreeCursor().destroy(); ctx.getBtreeCursor().close(); } return tokenExists; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/PartitionedTOccurrenceSearcher.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/PartitionedTOccurrenceSearcher.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/PartitionedTOccurrenceSearcher.java index ff5306c..5ae4e05 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/PartitionedTOccurrenceSearcher.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/PartitionedTOccurrenceSearcher.java @@ -92,16 +92,14 @@ public class PartitionedTOccurrenceSearcher extends AbstractTOccurrenceSearcher IPartitionedInvertedIndex partInvIndex = (IPartitionedInvertedIndex) invIndex; searchResult.reset(); if (partInvIndex.isEmpty()) { + resultCursor.open(null, searchPred); return; } - tokenizeQuery(searchPred); short numQueryTokens = (short) queryTokenAppender.getTupleCount(); - IInvertedIndexSearchModifier searchModifier = searchPred.getSearchModifier(); short numTokensLowerBound = searchModifier.getNumTokensLowerBound(numQueryTokens); short numTokensUpperBound = searchModifier.getNumTokensUpperBound(numQueryTokens); - occurrenceThreshold = searchModifier.getOccurrenceThreshold(numQueryTokens); if (occurrenceThreshold <= 0) { throw HyracksDataException.create(ErrorCode.OCCURRENCE_THRESHOLD_PANIC_EXCEPTION); @@ -118,15 +116,14 @@ public class PartitionedTOccurrenceSearcher extends AbstractTOccurrenceSearcher maxCountPossible--; // No results possible. if (maxCountPossible < occurrenceThreshold) { + resultCursor.open(null, searchPred); return; } } } - ArrayList<IInvertedListCursor>[] partitionCursors = partitions.getPartitions(); short start = partitions.getMinValidPartitionIndex(); short end = partitions.getMaxValidPartitionIndex(); - // Typically, we only enter this case for disk-based inverted indexes. // TODO: This behavior could potentially lead to a deadlock if we cannot pin // all inverted lists in memory, and are forced to wait for a page to get evicted @@ -165,7 +162,6 @@ public class PartitionedTOccurrenceSearcher extends AbstractTOccurrenceSearcher invListMerger.reset(); invListMerger.merge(partitionCursors[i], occurrenceThreshold, numPrefixLists, searchResult); } - resultCursor.open(null, searchPred); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/pom.xml index 856bf6b..1c118de 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/pom.xml @@ -16,17 +16,15 @@ ! specific language governing permissions and limitations ! under the License. !--> - -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> <modelVersion>4.0.0</modelVersion> <artifactId>hyracks-storage-am-lsm-rtree</artifactId> - <parent> <groupId>org.apache.hyracks</groupId> <artifactId>hyracks</artifactId> <version>0.3.4-SNAPSHOT</version> </parent> - <licenses> <license> <name>Apache License, Version 2.0</name> @@ -35,11 +33,9 @@ <comments>A business-friendly OSS license</comments> </license> </licenses> - <properties> <root.dir>${basedir}/../..</root.dir> </properties> - <dependencies> <dependency> <groupId>org.apache.hyracks</groupId> @@ -91,5 +87,9 @@ <artifactId>hyracks-storage-am-bloomfilter</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> + </dependency> </dependencies> -</project> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java index 09237a3..9be61fc 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java @@ -19,6 +19,7 @@ package org.apache.hyracks.storage.am.lsm.rtree.impls; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; +import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex; @@ -68,4 +69,9 @@ public class ExternalRTreeOpContext extends AbstractLSMIndexOperationContext { public LSMRTreeCursorInitialState getInitialState() { return initialState; } + + @Override + public void destroy() throws HyracksDataException { + // No Op + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java index 7172b74..d891d9e 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java @@ -22,6 +22,8 @@ package org.apache.hyracks.storage.am.lsm.rtree.impls; import java.util.ArrayList; import java.util.List; +import org.apache.commons.lang3.mutable.MutableBoolean; +import org.apache.commons.lang3.mutable.MutableLong; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory; import org.apache.hyracks.api.exceptions.ErrorCode; @@ -32,7 +34,6 @@ import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor; import org.apache.hyracks.storage.am.btree.impls.RangePredicate; import org.apache.hyracks.storage.am.common.api.IIndexOperationContext; -import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters; import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; @@ -115,124 +116,247 @@ public class LSMRTree extends AbstractLSMRTree { // Renaming order is critical because we use assume ordering when we // read the file names when we open the tree. // The RTree should be renamed before the BTree. - // scan the memory RTree - RTreeAccessor memRTreeAccessor = - flushingComponent.getIndex().createAccessor(NoOpIndexAccessParameters.INSTANCE); - RTreeSearchCursor rtreeScanCursor = memRTreeAccessor.createSearchCursor(false); - SearchPredicate rtreeNullPredicate = new SearchPredicate(null, null); - memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate); - - ILSMDiskComponent component = createDiskComponent(componentFactory, flushOp.getTarget(), - flushOp.getBTreeTarget(), flushOp.getBloomFilterTarget(), true); - - //count the number of tuples in the buddy btree + TreeTupleSorter rTreeTupleSorter = null; + MutableBoolean isEmpty = new MutableBoolean(true); + MutableLong numBTreeTuples = new MutableLong(0L); + ILSMDiskComponent component; + ILSMDiskComponentBulkLoader componentBulkLoader = null; + RangePredicate btreeNullPredicate = new RangePredicate(null, null, true, true, null, null); BTreeAccessor memBTreeAccessor = flushingComponent.getBuddyIndex().createAccessor(NoOpIndexAccessParameters.INSTANCE); - RangePredicate btreeNullPredicate = new RangePredicate(null, null, true, true, null, null); - IIndexCursor btreeCountingCursor = memBTreeAccessor.createCountingSearchCursor(); - memBTreeAccessor.search(btreeCountingCursor, btreeNullPredicate); - long numBTreeTuples = 0L; + boolean abort = true; try { - while (btreeCountingCursor.hasNext()) { - btreeCountingCursor.next(); - ITupleReference countTuple = btreeCountingCursor.getTuple(); - numBTreeTuples = IntegerPointable.getInteger(countTuple.getFieldData(0), countTuple.getFieldStart(0)); + try { + rTreeTupleSorter = getRTreeTupleSorter(flushingComponent, memBTreeAccessor, btreeNullPredicate, + numBTreeTuples, isEmpty); + rTreeTupleSorter.sort(); + component = createDiskComponent(componentFactory, flushOp.getTarget(), flushOp.getBTreeTarget(), + flushOp.getBloomFilterTarget(), true); + componentBulkLoader = + component.createBulkLoader(1.0f, false, numBTreeTuples.longValue(), false, false, false); + flushLoadRTree(isEmpty, rTreeTupleSorter, componentBulkLoader); + // scan the memory BTree and bulk load delete tuples + flushLoadBtree(memBTreeAccessor, componentBulkLoader, btreeNullPredicate); + } finally { + try { + memBTreeAccessor.destroy(); + } finally { + if (rTreeTupleSorter != null) { + rTreeTupleSorter.destroy(); + } + } } - } finally { - btreeCountingCursor.destroy(); - } - - ILSMDiskComponentBulkLoader componentBulkLoader = - component.createBulkLoader(1.0f, false, numBTreeTuples, false, false, false); - - ITreeIndexCursor cursor; - IBinaryComparatorFactory[] linearizerArray = { linearizer }; - - TreeTupleSorter rTreeTupleSorter = new TreeTupleSorter(flushingComponent.getIndex().getFileId(), - linearizerArray, rtreeLeafFrameFactory.createFrame(), rtreeLeafFrameFactory.createFrame(), - flushingComponent.getIndex().getBufferCache(), comparatorFields); - - // BulkLoad the tuples from the in-memory tree into the new disk - // RTree. - boolean isEmpty = true; - try { - while (rtreeScanCursor.hasNext()) { - isEmpty = false; - rtreeScanCursor.next(); - rTreeTupleSorter.insertTupleEntry(rtreeScanCursor.getPageId(), rtreeScanCursor.getTupleOffset()); + if (component.getLSMComponentFilter() != null) { + List<ITupleReference> filterTuples = new ArrayList<>(); + filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple()); + filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple()); + getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples); + getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getMetadataHolder()); } + // Note. If we change the filter to write to metadata object, we don't need the if block above + flushingComponent.getMetadata().copy(component.getMetadata()); + abort = false; + componentBulkLoader.end(); } finally { - rtreeScanCursor.destroy(); + if (abort && componentBulkLoader != null) { + componentBulkLoader.abort(); + } } - rTreeTupleSorter.sort(); - - cursor = rTreeTupleSorter; + return component; + } - if (!isEmpty) { + private void flushLoadRTree(MutableBoolean isEmpty, TreeTupleSorter rTreeTupleSorter, + ILSMDiskComponentBulkLoader componentBulkLoader) throws HyracksDataException { + if (!isEmpty.booleanValue()) { + rTreeTupleSorter.open(null, null); try { - while (cursor.hasNext()) { - cursor.next(); - ITupleReference frameTuple = cursor.getTuple(); + while (rTreeTupleSorter.hasNext()) { + rTreeTupleSorter.next(); + ITupleReference frameTuple = rTreeTupleSorter.getTuple(); componentBulkLoader.add(frameTuple); } } finally { - cursor.destroy(); + rTreeTupleSorter.close(); } } + } - // scan the memory BTree + private void flushLoadBtree(BTreeAccessor memBTreeAccessor, ILSMDiskComponentBulkLoader componentBulkLoader, + RangePredicate btreeNullPredicate) throws HyracksDataException { IIndexCursor btreeScanCursor = memBTreeAccessor.createSearchCursor(false); - memBTreeAccessor.search(btreeScanCursor, btreeNullPredicate); try { - while (btreeScanCursor.hasNext()) { - btreeScanCursor.next(); - ITupleReference frameTuple = btreeScanCursor.getTuple(); - componentBulkLoader.delete(frameTuple); + memBTreeAccessor.search(btreeScanCursor, btreeNullPredicate); + try { + while (btreeScanCursor.hasNext()) { + btreeScanCursor.next(); + ITupleReference frameTuple = btreeScanCursor.getTuple(); + componentBulkLoader.delete(frameTuple); + } + } finally { + btreeScanCursor.close(); } } finally { btreeScanCursor.destroy(); } + } - if (component.getLSMComponentFilter() != null) { - List<ITupleReference> filterTuples = new ArrayList<>(); - filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple()); - filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple()); - getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples); - getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getMetadataHolder()); + private TreeTupleSorter getRTreeTupleSorter(LSMRTreeMemoryComponent flushingComponent, + BTreeAccessor memBTreeAccessor, RangePredicate btreeNullPredicate, MutableLong numBTreeTuples, + MutableBoolean isEmpty) throws HyracksDataException { + RTreeAccessor memRTreeAccessor = + flushingComponent.getIndex().createAccessor(NoOpIndexAccessParameters.INSTANCE); + try { + RTreeSearchCursor rtreeScanCursor = memRTreeAccessor.createSearchCursor(false); + try { + SearchPredicate rtreeNullPredicate = new SearchPredicate(null, null); + memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate); + try { + //count the number of tuples in the buddy btree + countTuples(memBTreeAccessor, btreeNullPredicate, numBTreeTuples); + IBinaryComparatorFactory[] linearizerArray = { linearizer }; + boolean failed = true; + TreeTupleSorter rTreeTupleSorter = new TreeTupleSorter(flushingComponent.getIndex().getFileId(), + linearizerArray, rtreeLeafFrameFactory.createFrame(), rtreeLeafFrameFactory.createFrame(), + flushingComponent.getIndex().getBufferCache(), comparatorFields); + try { + // BulkLoad the tuples from the in-memory tree into the new disk + // RTree. + isEmpty.setValue(fill(rtreeScanCursor, rTreeTupleSorter)); + failed = false; + } finally { + if (failed) { + rTreeTupleSorter.destroy(); + } + } + return rTreeTupleSorter; + } finally { + rtreeScanCursor.close(); + } + } finally { + rtreeScanCursor.destroy(); + } + } finally { + memRTreeAccessor.destroy(); } - // Note. If we change the filter to write to metadata object, we don't need the if block above - flushingComponent.getMetadata().copy(component.getMetadata()); + } - componentBulkLoader.end(); - return component; + private boolean fill(RTreeSearchCursor rtreeScanCursor, TreeTupleSorter rTreeTupleSorter) + throws HyracksDataException { + boolean isEmpty = true; + while (rtreeScanCursor.hasNext()) { + isEmpty = false; + rtreeScanCursor.next(); + rTreeTupleSorter.insertTupleEntry(rtreeScanCursor.getPageId(), rtreeScanCursor.getTupleOffset()); + } + return isEmpty; + } + + private void countTuples(BTreeAccessor memBTreeAccessor, RangePredicate btreeNullPredicate, + MutableLong numBTreeTuples) throws HyracksDataException { + IIndexCursor btreeCountingCursor = memBTreeAccessor.createCountingSearchCursor(); + try { + memBTreeAccessor.search(btreeCountingCursor, btreeNullPredicate); + try { + while (btreeCountingCursor.hasNext()) { + btreeCountingCursor.next(); + ITupleReference countTuple = btreeCountingCursor.getTuple(); + numBTreeTuples.setValue( + IntegerPointable.getInteger(countTuple.getFieldData(0), countTuple.getFieldStart(0))); + } + } finally { + btreeCountingCursor.close(); + } + } finally { + btreeCountingCursor.destroy(); + } } @Override public ILSMDiskComponent doMerge(ILSMIOOperation operation) throws HyracksDataException { LSMRTreeMergeOperation mergeOp = (LSMRTreeMergeOperation) operation; IIndexCursor cursor = mergeOp.getCursor(); + ILSMDiskComponentBulkLoader componentBulkLoader = null; + ILSMDiskComponent mergedComponent = null; + boolean abort = true; + try { + mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(), mergeOp.getBTreeTarget(), + mergeOp.getBloomFilterTarget(), true); + componentBulkLoader = loadMergeBulkLoader(mergeOp, cursor, mergedComponent); + if (mergedComponent.getLSMComponentFilter() != null) { + List<ITupleReference> filterTuples = new ArrayList<>(); + for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) { + filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple()); + filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple()); + } + getFilterManager().updateFilter(mergedComponent.getLSMComponentFilter(), filterTuples); + getFilterManager().writeFilter(mergedComponent.getLSMComponentFilter(), + mergedComponent.getMetadataHolder()); + } + abort = false; + componentBulkLoader.end(); + } finally { + try { + cursor.destroy(); + } finally { + if (abort && componentBulkLoader != null) { + componentBulkLoader.abort(); + } + } + } + return mergedComponent; + } + + private ILSMDiskComponentBulkLoader loadMergeBulkLoader(LSMRTreeMergeOperation mergeOp, IIndexCursor cursor, + ILSMDiskComponent mergedComponent) throws HyracksDataException { + ILSMDiskComponentBulkLoader componentBulkLoader = null; + boolean abort = true; ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null); ILSMIndexOperationContext opCtx = ((LSMRTreeSortedCursor) cursor).getOpCtx(); search(opCtx, cursor, rtreeSearchPred); - - ILSMDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(), - mergeOp.getBTreeTarget(), mergeOp.getBloomFilterTarget(), true); - ILSMDiskComponentBulkLoader componentBulkLoader; - // In case we must keep the deleted-keys BTrees, then they must be merged *before* merging the r-trees so that - // lsmHarness.endSearch() is called once when the r-trees have been merged. - if (mergeOp.getMergingComponents().get(mergeOp.getMergingComponents().size() - 1) != diskComponents - .get(diskComponents.size() - 1)) { - // Keep the deleted tuples since the oldest disk component is not included in the merge operation - - long numElements = 0L; - for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) { - numElements += ((LSMRTreeDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter() - .getNumElements(); + try { + try { + // In case we must keep the deleted-keys BTrees, then they must be merged + // *before* merging the r-trees so that + // lsmHarness.endSearch() is called once when the r-trees have been merged. + if (mergeOp.getMergingComponents().get(mergeOp.getMergingComponents().size() - 1) != diskComponents + .get(diskComponents.size() - 1)) { + // Keep the deleted tuples since the oldest disk component + // is not included in the merge operation + long numElements = 0L; + for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) { + numElements += ((LSMRTreeDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter() + .getNumElements(); + } + componentBulkLoader = + mergedComponent.createBulkLoader(1.0f, false, numElements, false, false, false); + mergeLoadBTree(opCtx, rtreeSearchPred, componentBulkLoader); + } else { + //no buddy-btree needed + componentBulkLoader = mergedComponent.createBulkLoader(1.0f, false, 0L, false, false, false); + } + //search old rtree components + while (cursor.hasNext()) { + cursor.next(); + ITupleReference frameTuple = cursor.getTuple(); + componentBulkLoader.add(frameTuple); + } + } finally { + cursor.close(); + } + abort = false; + } finally { + if (abort && componentBulkLoader != null) { + componentBulkLoader.abort(); } - componentBulkLoader = mergedComponent.createBulkLoader(1.0f, false, numElements, false, false, false); + } + return componentBulkLoader; + } - LSMRTreeDeletedKeysBTreeMergeCursor btreeCursor = new LSMRTreeDeletedKeysBTreeMergeCursor(opCtx); + private void mergeLoadBTree(ILSMIndexOperationContext opCtx, ISearchPredicate rtreeSearchPred, + ILSMDiskComponentBulkLoader componentBulkLoader) throws HyracksDataException { + LSMRTreeDeletedKeysBTreeMergeCursor btreeCursor = new LSMRTreeDeletedKeysBTreeMergeCursor(opCtx); + try { search(opCtx, btreeCursor, rtreeSearchPred); try { while (btreeCursor.hasNext()) { @@ -241,38 +365,11 @@ public class LSMRTree extends AbstractLSMRTree { componentBulkLoader.delete(tuple); } } finally { - btreeCursor.destroy(); - } - } else { - //no buddy-btree needed - componentBulkLoader = mergedComponent.createBulkLoader(1.0f, false, 0L, false, false, false); - } - - //search old rtree components - try { - while (cursor.hasNext()) { - cursor.next(); - ITupleReference frameTuple = cursor.getTuple(); - componentBulkLoader.add(frameTuple); + btreeCursor.close(); } } finally { - cursor.destroy(); + btreeCursor.destroy(); } - - if (mergedComponent.getLSMComponentFilter() != null) { - List<ITupleReference> filterTuples = new ArrayList<>(); - for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) { - filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple()); - filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple()); - } - getFilterManager().updateFilter(mergedComponent.getLSMComponentFilter(), filterTuples); - getFilterManager().writeFilter(mergedComponent.getLSMComponentFilter(), - mergedComponent.getMetadataHolder()); - } - - componentBulkLoader.end(); - - return mergedComponent; } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java index e4267e2..d41e406 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java @@ -41,12 +41,12 @@ import org.apache.hyracks.storage.am.rtree.impls.RTree; import org.apache.hyracks.storage.am.rtree.impls.RTree.RTreeAccessor; import org.apache.hyracks.storage.am.rtree.impls.RTreeSearchCursor; import org.apache.hyracks.storage.am.rtree.impls.SearchPredicate; +import org.apache.hyracks.storage.common.EnforcedIndexCursor; import org.apache.hyracks.storage.common.ICursorInitialState; -import org.apache.hyracks.storage.common.IIndexCursor; import org.apache.hyracks.storage.common.ISearchPredicate; import org.apache.hyracks.storage.common.MultiComparator; -public abstract class LSMRTreeAbstractCursor implements ILSMIndexCursor { +public abstract class LSMRTreeAbstractCursor extends EnforcedIndexCursor implements ILSMIndexCursor { protected boolean open; protected RTreeSearchCursor[] rtreeCursors; @@ -76,7 +76,7 @@ public abstract class LSMRTreeAbstractCursor implements ILSMIndexCursor { } @Override - public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException { + public void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException { LSMRTreeCursorInitialState lsmInitialState = (LSMRTreeCursorInitialState) initialState; if (btreeCmp == null) { btreeCmp = lsmInitialState.getBTreeCmp(); @@ -153,7 +153,7 @@ public abstract class LSMRTreeAbstractCursor implements ILSMIndexCursor { } @Override - public void destroy() throws HyracksDataException { + public void doDestroy() throws HyracksDataException { if (!open) { return; } @@ -176,7 +176,7 @@ public abstract class LSMRTreeAbstractCursor implements ILSMIndexCursor { } @Override - public ITupleReference getTuple() { + public ITupleReference doGetTuple() { return frameTuple; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDeletedKeysBTreeMergeCursor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDeletedKeysBTreeMergeCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDeletedKeysBTreeMergeCursor.java index df4e16e..892fe83 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDeletedKeysBTreeMergeCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDeletedKeysBTreeMergeCursor.java @@ -46,7 +46,7 @@ public class LSMRTreeDeletedKeysBTreeMergeCursor extends LSMIndexSearchCursor { } @Override - public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException { + public void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException { LSMRTreeCursorInitialState lsmInitialState = (LSMRTreeCursorInitialState) initialState; cmp = lsmInitialState.getBTreeCmp(); operationalComponents = lsmInitialState.getOperationalComponents(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java index 50f1961..c3c74b8 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java @@ -22,6 +22,8 @@ package org.apache.hyracks.storage.am.lsm.rtree.impls; import java.util.List; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.util.DestroyUtils; import org.apache.hyracks.storage.am.btree.impls.BTree; import org.apache.hyracks.storage.am.btree.impls.BTreeOpContext; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; @@ -51,6 +53,7 @@ public final class LSMRTreeOpContext extends AbstractLSMIndexOperationContext { private RTreeOpContext currentRTreeOpContext; private BTreeOpContext currentBTreeOpContext; private LSMRTreeCursorInitialState searchInitialState; + private boolean destroyed = false; public LSMRTreeOpContext(ILSMIndex index, List<ILSMMemoryComponent> mutableComponents, ITreeIndexFrameFactory rtreeLeafFrameFactory, ITreeIndexFrameFactory rtreeInteriorFrameFactory, @@ -128,4 +131,19 @@ public final class LSMRTreeOpContext extends AbstractLSMIndexOperationContext { public RTreeOpContext getCurrentRTreeOpContext() { return currentRTreeOpContext; } + + @Override + public void destroy() throws HyracksDataException { + if (destroyed) { + return; + } + destroyed = true; + Throwable failure = DestroyUtils.destroy(null, mutableRTreeAccessors); + failure = DestroyUtils.destroy(failure, rtreeOpContexts); + failure = DestroyUtils.destroy(failure, mutableBTreeAccessors); + failure = DestroyUtils.destroy(failure, btreeOpContexts); + if (failure != null) { + throw HyracksDataException.create(failure); + } + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java index 0edf22f..d485f64 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java @@ -39,13 +39,13 @@ public class LSMRTreeSearchCursor extends LSMRTreeAbstractCursor { } @Override - public void destroy() throws HyracksDataException { - super.destroy(); + public void doDestroy() throws HyracksDataException { + super.doDestroy(); currentCursor = 0; } @Override - public void close() throws HyracksDataException { + public void doClose() throws HyracksDataException { if (!open) { return; } @@ -54,8 +54,8 @@ public class LSMRTreeSearchCursor extends LSMRTreeAbstractCursor { foundNext = false; try { for (int i = 0; i < numberOfTrees; i++) { - rtreeCursors[i].destroy(); - btreeCursors[i].destroy(); + rtreeCursors[i].close(); + btreeCursors[i].close(); } rtreeCursors = null; btreeCursors = null; @@ -88,7 +88,7 @@ public class LSMRTreeSearchCursor extends LSMRTreeAbstractCursor { } @Override - public boolean hasNext() throws HyracksDataException { + public boolean doHasNext() throws HyracksDataException { if (foundNext) { return true; } @@ -111,7 +111,7 @@ public class LSMRTreeSearchCursor extends LSMRTreeAbstractCursor { killerTupleFound = true; } } finally { - btreeCursors[i].destroy(); + btreeCursors[i].close(); } } if (!killerTupleFound) { @@ -120,7 +120,7 @@ public class LSMRTreeSearchCursor extends LSMRTreeAbstractCursor { return true; } } - rtreeCursors[currentCursor].destroy(); + rtreeCursors[currentCursor].close(); currentCursor++; searchNextCursor(); } @@ -128,13 +128,13 @@ public class LSMRTreeSearchCursor extends LSMRTreeAbstractCursor { } @Override - public void next() throws HyracksDataException { + public void doNext() throws HyracksDataException { foundNext = false; } @Override - public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException { - super.open(initialState, searchPred); + public void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException { + super.doOpen(initialState, searchPred); searchNextCursor(); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSortedCursor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSortedCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSortedCursor.java index d66e882..9bbc3e1 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSortedCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSortedCursor.java @@ -51,17 +51,21 @@ public class LSMRTreeSortedCursor extends LSMRTreeAbstractCursor { } @Override - public void close() throws HyracksDataException { + public void doClose() throws HyracksDataException { depletedRtreeCursors = new boolean[numberOfTrees]; foundNext = false; try { for (int i = 0; i < numberOfTrees; i++) { rtreeCursors[i].close(); rtreeAccessors[i].search(rtreeCursors[i], rtreeSearchPredicate); - if (rtreeCursors[i].hasNext()) { - rtreeCursors[i].next(); - } else { - depletedRtreeCursors[i] = true; + try { + if (rtreeCursors[i].hasNext()) { + rtreeCursors[i].next(); + } else { + depletedRtreeCursors[i] = true; + } + } finally { + rtreeCursors[i].close(); } } } finally { @@ -88,7 +92,7 @@ public class LSMRTreeSortedCursor extends LSMRTreeAbstractCursor { } @Override - public boolean hasNext() throws HyracksDataException { + public boolean doHasNext() throws HyracksDataException { while (!foundNext) { frameTuple = null; @@ -138,7 +142,7 @@ public class LSMRTreeSortedCursor extends LSMRTreeAbstractCursor { break; } } finally { - btreeCursors[i].destroy(); + btreeCursors[i].close(); } } if (!killed) { @@ -150,14 +154,13 @@ public class LSMRTreeSortedCursor extends LSMRTreeAbstractCursor { } @Override - public void next() throws HyracksDataException { + public void doNext() throws HyracksDataException { foundNext = false; } @Override - public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException { - super.open(initialState, searchPred); - + public void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException { + super.doOpen(initialState, searchPred); depletedRtreeCursors = new boolean[numberOfTrees]; foundNext = false; for (int i = 0; i < numberOfTrees; i++) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java index 94d76f6..cade80f 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java @@ -92,85 +92,133 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree { // read the file names when we open the tree. // The RTree should be renamed before the BTree. LSMRTreeMemoryComponent flushingComponent = (LSMRTreeMemoryComponent) flushOp.getFlushingComponent(); - RTreeAccessor memRTreeAccessor = - flushingComponent.getIndex().createAccessor(NoOpIndexAccessParameters.INSTANCE); - RTreeSearchCursor rtreeScanCursor = memRTreeAccessor.createSearchCursor(false); SearchPredicate rtreeNullPredicate = new SearchPredicate(null, null); - memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate); - ILSMDiskComponent component = createDiskComponent(componentFactory, flushOp.getTarget(), null, null, true); - ILSMDiskComponentBulkLoader componentBulkLoader = - component.createBulkLoader(1.0f, false, 0L, false, false, false); - - // Since the LSM-RTree is used as a secondary assumption, the - // primary key will be the last comparator in the BTree comparators - TreeTupleSorter rTreeTupleSorter = new TreeTupleSorter(flushingComponent.getIndex().getFileId(), - linearizerArray, rtreeLeafFrameFactory.createFrame(), rtreeLeafFrameFactory.createFrame(), - flushingComponent.getIndex().getBufferCache(), comparatorFields); - + ILSMDiskComponent component = null; + ILSMDiskComponentBulkLoader componentBulkLoader = null; + TreeTupleSorter rTreeTupleSorter = null; + TreeTupleSorter bTreeTupleSorter = null; boolean isEmpty = true; + boolean abort = true; try { - while (rtreeScanCursor.hasNext()) { - isEmpty = false; - rtreeScanCursor.next(); - rTreeTupleSorter.insertTupleEntry(rtreeScanCursor.getPageId(), rtreeScanCursor.getTupleOffset()); - } - } finally { - rtreeScanCursor.destroy(); - } - if (!isEmpty) { - rTreeTupleSorter.sort(); - } + RTreeAccessor memRTreeAccessor = + flushingComponent.getIndex().createAccessor(NoOpIndexAccessParameters.INSTANCE); - // scan the memory BTree - BTreeAccessor memBTreeAccessor = - flushingComponent.getBuddyIndex().createAccessor(NoOpIndexAccessParameters.INSTANCE); - BTreeRangeSearchCursor btreeScanCursor = memBTreeAccessor.createSearchCursor(false); - RangePredicate btreeNullPredicate = new RangePredicate(null, null, true, true, null, null); - memBTreeAccessor.search(btreeScanCursor, btreeNullPredicate); - TreeTupleSorter bTreeTupleSorter = new TreeTupleSorter(flushingComponent.getBuddyIndex().getFileId(), - linearizerArray, btreeLeafFrameFactory.createFrame(), btreeLeafFrameFactory.createFrame(), - flushingComponent.getBuddyIndex().getBufferCache(), comparatorFields); - - isEmpty = true; - try { - while (btreeScanCursor.hasNext()) { - isEmpty = false; - btreeScanCursor.next(); - bTreeTupleSorter.insertTupleEntry(btreeScanCursor.getPageId(), btreeScanCursor.getTupleOffset()); + try { + RTreeSearchCursor rtreeScanCursor = memRTreeAccessor.createSearchCursor(false); + try { + memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate); + component = createDiskComponent(componentFactory, flushOp.getTarget(), null, null, true); + componentBulkLoader = component.createBulkLoader(1.0f, false, 0L, false, false, false); + // Since the LSM-RTree is used as a secondary assumption, the + // primary key will be the last comparator in the BTree comparators + rTreeTupleSorter = new TreeTupleSorter(flushingComponent.getIndex().getFileId(), linearizerArray, + rtreeLeafFrameFactory.createFrame(), rtreeLeafFrameFactory.createFrame(), + flushingComponent.getIndex().getBufferCache(), comparatorFields); + try { + isEmpty = scanAndSort(rtreeScanCursor, rTreeTupleSorter); + } finally { + rtreeScanCursor.close(); + } + } finally { + rtreeScanCursor.destroy(); + } + } finally { + memRTreeAccessor.destroy(); } - } finally { - btreeScanCursor.destroy(); - } - if (!isEmpty) { - bTreeTupleSorter.sort(); - } - - LSMRTreeWithAntiMatterTuplesFlushCursor cursor = new LSMRTreeWithAntiMatterTuplesFlushCursor(rTreeTupleSorter, - bTreeTupleSorter, comparatorFields, linearizerArray); - cursor.open(null, null); - - try { - while (cursor.hasNext()) { - cursor.next(); - ITupleReference frameTuple = cursor.getTuple(); - - componentBulkLoader.add(frameTuple); + if (!isEmpty) { + rTreeTupleSorter.sort(); + } + // scan the memory BTree + RangePredicate btreeNullPredicate = new RangePredicate(null, null, true, true, null, null); + BTreeAccessor memBTreeAccessor = + flushingComponent.getBuddyIndex().createAccessor(NoOpIndexAccessParameters.INSTANCE); + try { + bTreeTupleSorter = new TreeTupleSorter(flushingComponent.getBuddyIndex().getFileId(), linearizerArray, + btreeLeafFrameFactory.createFrame(), btreeLeafFrameFactory.createFrame(), + flushingComponent.getBuddyIndex().getBufferCache(), comparatorFields); + BTreeRangeSearchCursor btreeScanCursor = memBTreeAccessor.createSearchCursor(false); + try { + isEmpty = true; + memBTreeAccessor.search(btreeScanCursor, btreeNullPredicate); + try { + isEmpty = scanAndSort(btreeScanCursor, bTreeTupleSorter); + } finally { + btreeScanCursor.close(); + } + } finally { + btreeScanCursor.destroy(); + } + } finally { + memBTreeAccessor.destroy(); + } + if (!isEmpty) { + bTreeTupleSorter.sort(); + } + LSMRTreeWithAntiMatterTuplesFlushCursor cursor = new LSMRTreeWithAntiMatterTuplesFlushCursor( + rTreeTupleSorter, bTreeTupleSorter, comparatorFields, linearizerArray); + try { + cursor.open(null, null); + try { + while (cursor.hasNext()) { + cursor.next(); + ITupleReference frameTuple = cursor.getTuple(); + componentBulkLoader.add(frameTuple); + } + } finally { + cursor.close(); + } + } finally { + cursor.destroy(); + } + if (component.getLSMComponentFilter() != null) { + List<ITupleReference> filterTuples = new ArrayList<>(); + filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple()); + filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple()); + getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples); + getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getMetadataHolder()); } + flushingComponent.getMetadata().copy(component.getMetadata()); + abort = false; + componentBulkLoader.end(); } finally { - cursor.destroy(); + try { + if (rTreeTupleSorter != null) { + rTreeTupleSorter.destroy(); + } + } finally { + try { + if (bTreeTupleSorter != null) { + bTreeTupleSorter.destroy(); + } + } finally { + if (abort && componentBulkLoader != null) { + componentBulkLoader.abort(); + } + } + } } + return component; + } - if (component.getLSMComponentFilter() != null) { - List<ITupleReference> filterTuples = new ArrayList<>(); - filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple()); - filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple()); - getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples); - getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getMetadataHolder()); + private boolean scanAndSort(RTreeSearchCursor scanCursor, TreeTupleSorter tupleSorter) throws HyracksDataException { + boolean isEmpty = true; + while (scanCursor.hasNext()) { + isEmpty = false; + scanCursor.next(); + tupleSorter.insertTupleEntry(scanCursor.getPageId(), scanCursor.getTupleOffset()); } - flushingComponent.getMetadata().copy(component.getMetadata()); + return isEmpty; + } - componentBulkLoader.end(); - return component; + private boolean scanAndSort(BTreeRangeSearchCursor scanCursor, TreeTupleSorter tupleSorter) + throws HyracksDataException { + boolean isEmpty = true; + while (scanCursor.hasNext()) { + isEmpty = false; + scanCursor.next(); + tupleSorter.insertTupleEntry(scanCursor.getPageId(), scanCursor.getTupleOffset()); + } + return isEmpty; } @Override @@ -193,7 +241,7 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree { componentBulkLoader.add(frameTuple); } } finally { - cursor.destroy(); + cursor.close(); } if (component.getLSMComponentFilter() != null) { List<ITupleReference> filterTuples = new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFlushCursor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFlushCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFlushCursor.java index c2a2bf5..77bf58e 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFlushCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFlushCursor.java @@ -23,12 +23,12 @@ import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; import org.apache.hyracks.storage.am.common.api.ILSMIndexCursor; +import org.apache.hyracks.storage.common.EnforcedIndexCursor; import org.apache.hyracks.storage.common.ICursorInitialState; -import org.apache.hyracks.storage.common.IIndexCursor; import org.apache.hyracks.storage.common.ISearchPredicate; import org.apache.hyracks.storage.common.MultiComparator; -public class LSMRTreeWithAntiMatterTuplesFlushCursor implements ILSMIndexCursor { +public class LSMRTreeWithAntiMatterTuplesFlushCursor extends EnforcedIndexCursor implements ILSMIndexCursor { private final TreeTupleSorter rTreeTupleSorter; private final TreeTupleSorter bTreeTupleSorter; private final int[] comparatorFields; @@ -48,12 +48,23 @@ public class LSMRTreeWithAntiMatterTuplesFlushCursor implements ILSMIndexCursor } @Override - public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException { - + public void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException { + boolean rtreeOpen = false; + boolean btreeOpen = false; + try { + rTreeTupleSorter.open(initialState, searchPred); + rtreeOpen = true; + bTreeTupleSorter.open(initialState, searchPred); + btreeOpen = true; + } finally { + if (rtreeOpen && !btreeOpen) { + rTreeTupleSorter.close(); + } + } } @Override - public boolean hasNext() throws HyracksDataException { + public boolean doHasNext() throws HyracksDataException { if (foundNext) { return true; } @@ -124,22 +135,31 @@ public class LSMRTreeWithAntiMatterTuplesFlushCursor implements ILSMIndexCursor } @Override - public void next() throws HyracksDataException { + public void doNext() throws HyracksDataException { foundNext = false; } @Override - public void destroy() throws HyracksDataException { + public void doDestroy() throws HyracksDataException { + try { + rTreeTupleSorter.destroy(); + } finally { + bTreeTupleSorter.destroy(); + } } @Override - public void close() throws HyracksDataException { - + public void doClose() throws HyracksDataException { + try { + rTreeTupleSorter.close(); + } finally { + bTreeTupleSorter.close(); + } } @Override - public ITupleReference getTuple() { + public ITupleReference doGetTuple() { return frameTuple; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java index b8180a6..4547063 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java @@ -70,7 +70,7 @@ public class LSMRTreeWithAntiMatterTuplesSearchCursor extends LSMIndexSearchCurs } @Override - public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException { + public void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException { LSMRTreeCursorInitialState lsmInitialState = (LSMRTreeCursorInitialState) initialState; cmp = lsmInitialState.getHilbertCmp(); btreeCmp = lsmInitialState.getBTreeCmp(); @@ -136,7 +136,7 @@ public class LSMRTreeWithAntiMatterTuplesSearchCursor extends LSMIndexSearchCurs } @Override - public boolean hasNext() throws HyracksDataException { + public boolean doHasNext() throws HyracksDataException { if (numMemoryComponents > 0) { if (foundNext) { return true; @@ -158,13 +158,13 @@ public class LSMRTreeWithAntiMatterTuplesSearchCursor extends LSMIndexSearchCurs return true; } } - mutableRTreeCursors[currentCursor].destroy(); + mutableRTreeCursors[currentCursor].close(); currentCursor++; searchNextCursor(); } - while (super.hasNext()) { - super.next(); - ITupleReference diskRTreeTuple = super.getTuple(); + while (super.doHasNext()) { + super.doNext(); + ITupleReference diskRTreeTuple = super.doGetTuple(); // TODO: at this time, we only add proceed(). // reconcile() and complete() can be added later after considering the semantics. @@ -178,13 +178,11 @@ public class LSMRTreeWithAntiMatterTuplesSearchCursor extends LSMIndexSearchCurs } } } else { - if (super.hasNext()) { - super.next(); - ITupleReference diskRTreeTuple = super.getTuple(); - + if (super.doHasNext()) { + super.doNext(); + ITupleReference diskRTreeTuple = super.doGetTuple(); // TODO: at this time, we only add proceed() part. // reconcile() and complete() can be added later after considering the semantics. - // Call proceed() to do necessary operations before returning this tuple. // Since in-memory components don't exist, we can skip searching in-memory B-Trees. searchCallback.proceed(diskRTreeTuple); @@ -214,17 +212,17 @@ public class LSMRTreeWithAntiMatterTuplesSearchCursor extends LSMIndexSearchCurs } @Override - public void next() throws HyracksDataException { + public void doNext() throws HyracksDataException { foundNext = false; } @Override - public ITupleReference getTuple() { + public ITupleReference doGetTuple() { return frameTuple; } @Override - public void close() throws HyracksDataException { + public void doClose() throws HyracksDataException { if (!open) { return; } @@ -236,11 +234,11 @@ public class LSMRTreeWithAntiMatterTuplesSearchCursor extends LSMIndexSearchCurs btreeCursors[i].close(); } } - super.close(); + super.doClose(); } @Override - public void destroy() throws HyracksDataException { + public void doDestroy() throws HyracksDataException { if (!open) { return; } @@ -252,7 +250,7 @@ public class LSMRTreeWithAntiMatterTuplesSearchCursor extends LSMIndexSearchCurs } currentCursor = 0; open = false; - super.destroy(); + super.doDestroy(); } @Override @@ -272,7 +270,7 @@ public class LSMRTreeWithAntiMatterTuplesSearchCursor extends LSMIndexSearchCurs return false; } } finally { - btreeCursors[i].destroy(); + btreeCursors[i].close(); } } return true;