>From Ritik Raj <[email protected]>: Ritik Raj has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19990 )
Change subject: [ASTERIXDB-3601][STO] Properly unpinning segment files ...................................................................... [ASTERIXDB-3601][STO] Properly unpinning segment files - user model changes: no - storage format changes: no - interface changes: no Details: Properly unpinning the segment files, in case of failure. Otherwise it might cause resource leak. Ext-ref: MB-66306 Change-Id: I1dec880871008fc6b9be6662df9fa034992858c9 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19990 Reviewed-by: Peeyush Gupta <[email protected]> Integration-Tests: Jenkins <[email protected]> Tested-by: Ritik Raj <[email protected]> --- M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/SparseColumnMultiPageZeroReader.java M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/AbstractColumnMultiPageZeroReader.java M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/MultiPersistentPageZeroBufferBytesOutputStream.java M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/readers/DefaultColumnPageZeroReader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweeper.java M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/DefaultColumnMultiPageZeroReader.java M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/PageZeroWriterFlavorSelector.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IColumnPageZeroReader.java M asterixdb/asterix-app/src/test/resources/cc-single-partition.conf M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeReadLeafFrame.java 10 files changed, 97 insertions(+), 4 deletions(-) Approvals: Peeyush Gupta: Looks good to me, approved Ritik Raj: Verified Anon. E. Moose #1000171: Jenkins: Verified diff --git a/asterixdb/asterix-app/src/test/resources/cc-single-partition.conf b/asterixdb/asterix-app/src/test/resources/cc-single-partition.conf index d5837ad..23071d6 100644 --- a/asterixdb/asterix-app/src/test/resources/cc-single-partition.conf +++ b/asterixdb/asterix-app/src/test/resources/cc-single-partition.conf @@ -53,4 +53,4 @@ compiler.ordered.fields=false messaging.frame.size=4096 messaging.frame.count=512 -storage.buffercache.pagesize=32KB +storage.buffercache.pagesize=32KB \ No newline at end of file diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/MultiPersistentPageZeroBufferBytesOutputStream.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/MultiPersistentPageZeroBufferBytesOutputStream.java index c156e49..4ae0153 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/MultiPersistentPageZeroBufferBytesOutputStream.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/MultiPersistentPageZeroBufferBytesOutputStream.java @@ -42,7 +42,7 @@ @Override protected void preReset() { - if (allocatedBytes > 0) { + if (allocatedBytes > 0 || !buffers.isEmpty()) { //This should not be the case, with the pageZero segments. //As the stream should be finished after the flush. throw new IllegalStateException("PageZero segments should already be finished after flush"); diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/PageZeroWriterFlavorSelector.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/PageZeroWriterFlavorSelector.java index 06e441a..831e459 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/PageZeroWriterFlavorSelector.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/PageZeroWriterFlavorSelector.java @@ -44,12 +44,12 @@ * This class implements an optimization strategy for sparse columns: * - Default writer: Allocates space for all columns in the schema (suitable for dense data) * - Sparse writer: Only allocates space for present columns (suitable for sparse data) + * - Multi-page writers: Variants of the above writers that support multi-page operations *</p> * The selector automatically chooses the most space-efficient option based on the actual * space requirements of each approach. */ public class PageZeroWriterFlavorSelector implements IColumnPageZeroWriterFlavorSelector { - // Flag indicating which writer type is currently selected (DEFAULT_WRITER_FLAG=default, SPARSE_WRITER_FLAG=sparse) protected byte writerFlag = IColumnPageZeroWriter.ColumnPageZeroWriterType.ADAPTIVE.getWriterFlag(); // Cache of writer instances to avoid repeated object creation diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/readers/DefaultColumnPageZeroReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/readers/DefaultColumnPageZeroReader.java index 1692e70..a34d8c1 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/readers/DefaultColumnPageZeroReader.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/readers/DefaultColumnPageZeroReader.java @@ -18,6 +18,7 @@ */ package org.apache.asterix.column.zero.readers; +import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.FLAG_OFFSET; import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.LEFT_MOST_KEY_OFFSET; import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.MEGA_LEAF_NODE_LENGTH; import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.NEXT_LEAF_OFFSET; @@ -31,9 +32,16 @@ import org.apache.asterix.column.zero.writers.DefaultColumnPageZeroWriter; import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider; import org.apache.hyracks.storage.am.lsm.btree.column.cloud.IntPairUtil; +import org.apache.hyracks.storage.am.lsm.btree.column.error.ColumnarValueException; import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroReader; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.fasterxml.jackson.databind.node.ObjectNode; public class DefaultColumnPageZeroReader implements IColumnPageZeroReader { + protected static Logger LOGGER = LogManager.getLogger(); + protected ByteBuffer pageZeroBuf; protected BitSet pageZeroSegmentsPages; protected int numberOfPresentColumns; @@ -182,4 +190,14 @@ pageZeroSegmentsPages.set(0); return pageZeroSegmentsPages; } + + @Override + public void printPageZeroReaderInfo() { + ColumnarValueException ex = new ColumnarValueException(); + ObjectNode readerNode = ex.createNode(getClass().getSimpleName()); + readerNode.put("headerSize", headerSize); + readerNode.put("numberOfPresentColumns", numberOfPresentColumns); + readerNode.put("flag", pageZeroBuf.get(FLAG_OFFSET)); + LOGGER.debug("SingleColumnPageZeroReader Info: {}", readerNode.toPrettyString()); + } } diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/AbstractColumnMultiPageZeroReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/AbstractColumnMultiPageZeroReader.java index 3b05fc3..b0fc721 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/AbstractColumnMultiPageZeroReader.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/AbstractColumnMultiPageZeroReader.java @@ -20,8 +20,11 @@ import org.apache.asterix.column.bytes.stream.in.MultiPageZeroByteBuffersReader; import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroReader; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; public abstract class AbstractColumnMultiPageZeroReader implements IColumnPageZeroReader { + protected static final Logger LOGGER = LogManager.getLogger(); protected MultiPageZeroByteBuffersReader segmentBuffers; AbstractColumnMultiPageZeroReader() { diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/DefaultColumnMultiPageZeroReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/DefaultColumnMultiPageZeroReader.java index db800ae..5a0b180 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/DefaultColumnMultiPageZeroReader.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/DefaultColumnMultiPageZeroReader.java @@ -40,6 +40,9 @@ import org.apache.hyracks.data.std.primitive.VoidPointable; import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider; import org.apache.hyracks.storage.am.lsm.btree.column.cloud.IntPairUtil; +import org.apache.hyracks.storage.am.lsm.btree.column.error.ColumnarValueException; + +import com.fasterxml.jackson.databind.node.ObjectNode; public class DefaultColumnMultiPageZeroReader extends AbstractColumnMultiPageZeroReader { public static final int headerSize = DefaultColumnMultiPageZeroWriter.EXTENDED_HEADER_SIZE; @@ -273,4 +276,15 @@ public int getHeaderSize() { return EXTENDED_HEADER_SIZE; } + + @Override + public void printPageZeroReaderInfo() { + ColumnarValueException ex = new ColumnarValueException(); + ObjectNode readerNode = ex.createNode(getClass().getSimpleName()); + readerNode.put("headerSize", headerSize); + readerNode.put("maxColumnsInZerothSegment", zerothSegmentMaxColumns); + readerNode.put("maxNumberOfColumnsInAPage", maxNumberOfColumnsInAPage); + readerNode.put("numberOfPageZeroSegments", numberOfPageZeroSegments); + LOGGER.debug("DefaultColumnMultiPageZeroReader Info: {}", readerNode.toPrettyString()); + } } diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/SparseColumnMultiPageZeroReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/SparseColumnMultiPageZeroReader.java index d555bdc..0a0a817 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/SparseColumnMultiPageZeroReader.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/SparseColumnMultiPageZeroReader.java @@ -40,6 +40,9 @@ import org.apache.hyracks.data.std.primitive.VoidPointable; import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider; import org.apache.hyracks.storage.am.lsm.btree.column.cloud.IntPairUtil; +import org.apache.hyracks.storage.am.lsm.btree.column.error.ColumnarValueException; + +import com.fasterxml.jackson.databind.node.ObjectNode; import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap; @@ -358,4 +361,16 @@ } return pageZeroSegmentsPages; } + + @Override + public void printPageZeroReaderInfo() { + ColumnarValueException ex = new ColumnarValueException(); + ObjectNode readerNode = ex.createNode(getClass().getSimpleName()); + readerNode.put("headerSize", headerSize); + readerNode.put("maxColumnIndexInZerothSegment", maxColumnIndexInZerothSegment); + readerNode.put("numberOfColumnInZerothSegment", numberOfColumnInZerothSegment); + readerNode.put("maxNumberOfColumnsInAPage", maxNumberOfColumnsInAPage); + readerNode.put("numberOfPageZeroSegments", numberOfPageZeroSegments); + LOGGER.debug("SparseColumnMultiPageZeroReader Info: {}", readerNode.toPrettyString()); + } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweeper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweeper.java index 1f0c7f4..ba4376f 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweeper.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweeper.java @@ -165,6 +165,7 @@ int freedSpace = 0; context.open(fileId); try { + Throwable failure = null; while (nextPageId >= 0) { if (context.stopSweeping()) { // Exit as the index is being dropped @@ -183,17 +184,30 @@ ranges.reset(leafFrame, plan); freedSpace += punchHoles(context, leafFrame); } + } catch (Throwable th) { + failure = th; } finally { if (columnsLocked) { page0.sweepUnlock(); } // unpin segment pages for (ICachedPage cachedPage : segmentPagesTempHolder) { - context.unpin(cachedPage, bcOpCtx); + try { + context.unpin(cachedPage, bcOpCtx); + } catch (Throwable e) { + if (failure == null) { + failure = e; + } else { + failure.addSuppressed(e); + } + } } context.unpin(page0, bcOpCtx); } } + if (failure != null) { + throw HyracksDataException.create(failure); + } } finally { context.close(); } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeReadLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeReadLeafFrame.java index 4a78400..c3f8228 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeReadLeafFrame.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeReadLeafFrame.java @@ -188,4 +188,8 @@ public BitSet markRequiredPageZeroSegments(BitSet projectedColumns, int pageZeroId, boolean markAll) { return columnPageZeroReader.markRequiredPageSegments(projectedColumns, pageZeroId, markAll); } + + public void printPageZeroReaderInfo() { + columnPageZeroReader.printPageZeroReaderInfo(); + } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IColumnPageZeroReader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IColumnPageZeroReader.java index db4f778..d9cf11b 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IColumnPageZeroReader.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IColumnPageZeroReader.java @@ -75,4 +75,6 @@ void resetStream(IColumnBufferProvider pageZeroSegmentBufferProvider) throws HyracksDataException; BitSet markRequiredPageSegments(BitSet projectedColumns, int pageZeroId, boolean markAll); + + void printPageZeroReaderInfo(); } -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19990 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: ionic Gerrit-Change-Id: I1dec880871008fc6b9be6662df9fa034992858c9 Gerrit-Change-Number: 19990 Gerrit-PatchSet: 5 Gerrit-Owner: Ritik Raj <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Peeyush Gupta <[email protected]> Gerrit-Reviewer: Ritik Raj <[email protected]> Gerrit-MessageType: merged
