Ian Maxon has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/575
Change subject: Fix for ASTERIXDB-1247
......................................................................
Fix for ASTERIXDB-1247
It seems like the root of this is the testing harness closing the index before
it has
had a chance to flush all of its pages. There are also some changes to cover
potential
corner cases where confiscated pages could be lost, but this doesn't seem to
directly
affect the bug.
Change-Id: Ia580242b3f7753fc2f793f879332de3270ee3fee
---
M
hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
M
hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
M
hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
M
hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
M
hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
5 files changed, 38 insertions(+), 16 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/hyracks refs/changes/75/575/1
diff --git
a/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
b/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
index 8c192a1..369dd2f 100644
---
a/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
+++
b/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
@@ -945,7 +945,6 @@
public class BTreeBulkLoader extends
AbstractTreeIndex.AbstractTreeIndexBulkLoader {
protected final ISplitKey splitKey;
protected final boolean verifyInput;
- protected List<ICachedPage> pagesToWrite;
public BTreeBulkLoader(float fillFactor, boolean verifyInput, boolean
appendOnly) throws TreeIndexException,
HyracksDataException {
@@ -953,7 +952,6 @@
this.verifyInput = verifyInput;
splitKey = new
BTreeSplitKey(leafFrame.getTupleWriter().createTupleReference());
splitKey.getTuple().setFieldCount(cmp.getKeyFieldCount());
- pagesToWrite = new ArrayList<ICachedPage>();
}
@Override
@@ -1013,13 +1011,7 @@
leafFrame.setPage(leafFrontier.page);
((IBTreeLeafFrame) leafFrame).insertSorted(tuple);
- } catch (IndexException e) {
- handleException();
- throw e;
- } catch (HyracksDataException e) {
- handleException();
- throw e;
- } catch (RuntimeException e) {
+ } catch (IndexException | HyracksDataException | RuntimeException
e) {
handleException();
throw e;
}
@@ -1116,8 +1108,13 @@
@Override
public void end() throws HyracksDataException {
- persistFrontiers(0, -1);
- super.end();
+ try{
+ persistFrontiers(0, -1);
+ super.end();
+ } catch ( HyracksDataException | RuntimeException e) {
+ handleException();
+ throw e;
+ }
}
@Override
diff --git
a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
index 015404d..a56dfdb 100644
---
a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
+++
b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
@@ -20,6 +20,7 @@
package org.apache.hyracks.storage.am.common.impls;
import java.util.ArrayList;
+import java.util.List;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -317,6 +318,7 @@
protected boolean releasedLatches;
public boolean appendOnly = false;
protected final IFIFOPageQueue queue;
+ protected List<ICachedPage> pagesToWrite;
public AbstractTreeIndexBulkLoader(float fillFactor, boolean
appendOnly) throws TreeIndexException,
HyracksDataException {
@@ -346,8 +348,8 @@
NodeFrontier leafFrontier = new
NodeFrontier(leafFrame.createTupleReference());
leafFrontier.pageId = freePageManager.getFreePage(metaFrame);
- leafFrontier.page = bufferCache.confiscatePage(BufferedFileHandle
- .getDiskPageId(fileId, leafFrontier.pageId));
+ leafFrontier.page = bufferCache.confiscatePage(
+ BufferedFileHandle.getDiskPageId(fileId,
leafFrontier.pageId));
interiorFrame.setPage(leafFrontier.page);
interiorFrame.initBuffer((byte) 0);
@@ -359,6 +361,7 @@
slotSize = leafFrame.getSlotSize();
nodeFrontiers.add(leafFrontier);
+ pagesToWrite = new ArrayList<>();
}
public abstract void add(ITupleReference tuple) throws IndexException,
HyracksDataException;
@@ -371,6 +374,9 @@
bufferCache.returnPage(frontierPage,false);
}
}
+ for(ICachedPage pageToDiscard: pagesToWrite){
+ bufferCache.returnPage(pageToDiscard);
+ }
releasedLatches = true;
}
diff --git
a/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
b/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
index ada54d2..a0062b4 100644
---
a/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
+++
b/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
@@ -877,7 +877,6 @@
ITreeIndexTupleReference mbrTuple =
interiorFrame.createTupleReference();
ByteBuffer mbr;
List<Integer> prevNodeFrontierPages = new ArrayList<Integer>();
- List<ICachedPage> pagesToWrite = new ArrayList<ICachedPage>();
public RTreeBulkLoader(float fillFactor, boolean appendOnly) throws
TreeIndexException, HyracksDataException {
super(fillFactor, appendOnly);
diff --git
a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
index f66726b..eaed9f6 100644
---
a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
+++
b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
@@ -1197,4 +1197,15 @@
}
}
+ public boolean confiscatedPagesRemain(){
+ synchronized (cachedPages){
+ for(ICachedPageInternal c: cachedPages){
+ if(c.confiscated()){
+ return false;
+ }
+ }
+ }
+ return false;
+ }
+
}
diff --git
a/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
b/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
index 8c01105..f77b7e6 100644
---
a/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
+++
b/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
@@ -25,6 +25,7 @@
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.apache.hyracks.storage.common.buffercache.BufferCache;
import org.junit.Test;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -659,6 +660,9 @@
treeIndex.validate();
treeIndex.deactivate();
treeIndex.destroy();
+ if(
((BufferCache)(treeIndex.getBufferCache())).confiscatedPagesRemain()){
+ fail();
+ }
}
/**
@@ -719,21 +723,26 @@
try {
bulkLoader.add(tuple);
} catch (UnsortedInputException e) {
- if (j != i) {
+ if (j != i ||
((BufferCache)(treeIndex.getBufferCache())).confiscatedPagesRemain()) {
fail("Unexpected exception: " + e.getMessage());
}
// Success.
+
break;
} catch (TreeIndexDuplicateKeyException e2) {
- if (j != i) {
+ if (j != i ||
((BufferCache)(treeIndex.getBufferCache())).confiscatedPagesRemain() ) {
fail("Unexpected exception: " + e2.getMessage());
}
// Success.
break;
}
}
+ if(
((BufferCache)(treeIndex.getBufferCache())).confiscatedPagesRemain()){
+ fail();
+ }
treeIndex.deactivate();
+ treeIndex.getBufferCache().finishQueue();
treeIndex.destroy();
}
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/575
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ia580242b3f7753fc2f793f879332de3270ee3fee
Gerrit-PatchSet: 1
Gerrit-Project: hyracks
Gerrit-Branch: master
Gerrit-Owner: Ian Maxon <[email protected]>
