Repository: asterixdb Updated Branches: refs/heads/master ac1b93529 -> b4f663115
[NO ISSUE][STO] Let ILSMIOOperations clean up for themselves - user model changes: no - storage format changes: no - interface changes: yes remove ILSMIndex.cleanUpFilesForFailedOperation remove ILSMIOOperation.getComponentFiles add ILSMIOOperation.cleanup Details: Instead of having ILSMIndex instances clean up for ILSMIOOperations, let the ILSMIOOperations cleanup their component files. Also avoids a dependency of the interface ILSMIOOperation on o.a.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences. Change-Id: I97a31d018139dcdea15196ae52d91eed5e780fbd Reviewed-on: https://asterix-gerrit.ics.uci.edu/3034 Sonar-Qube: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Contrib: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: Murtadha Hubail <mhub...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/b4f66311 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/b4f66311 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/b4f66311 Branch: refs/heads/master Commit: b4f66311584ddb42470d0e185a477282f4aaf73c Parents: ac1b935 Author: Till Westmann <ti...@apache.org> Authored: Sat Nov 17 19:47:47 2018 -0800 Committer: Till Westmann <ti...@apache.org> Committed: Sun Nov 18 09:48:11 2018 -0800 ---------------------------------------------------------------------- .../am/lsm/common/api/ILSMIOOperation.java | 9 +++++--- .../storage/am/lsm/common/api/ILSMIndex.java | 7 ------ .../lsm/common/impls/AbstractIoOperation.java | 23 ++++++++++++++++++++ .../am/lsm/common/impls/AbstractLSMIndex.java | 19 ---------------- .../storage/am/lsm/common/impls/LSMHarness.java | 2 +- .../impls/LSMIndexDiskComponentBulkLoader.java | 2 +- .../am/lsm/common/impls/NoOpIoOperation.java | 3 ++- .../am/lsm/common/impls/TracedIOOperation.java | 5 +++-- 8 files changed, 36 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4f66311/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java index 0e13933..79463e5 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java @@ -25,7 +25,7 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IODeviceHandle; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation.LSMIOOperationStatus; -import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; +import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.IPageWriteFailureCallback; public interface ILSMIOOperation extends Callable<LSMIOOperationStatus>, IPageWriteFailureCallback { @@ -88,9 +88,12 @@ public interface ILSMIOOperation extends Callable<LSMIOOperationStatus>, IPageWr ILSMIndexAccessor getAccessor(); /** - * @return the component files produced by this operation + * clean up left over files in case of an exception during execution + * + * @param bufferCache + * a buffercache that manages the files */ - LSMComponentFileReferences getComponentFiles(); + void cleanup(IBufferCache bufferCache); /** * @return the failure in the io operation if any, null otherwise http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4f66311/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java index 5cb05a7..23b3634 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java @@ -181,13 +181,6 @@ public interface ILSMIndex extends IIndex { ILSMHarness getHarness(); /** - * Cleanup the files of the failed operation - * - * @param operation - */ - void cleanUpFilesForFailedOperation(ILSMIOOperation operation); - - /** * @return the absolute path of the index */ String getIndexIdentifier(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4f66311/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java index 3d76755..38ef179 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java @@ -22,6 +22,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IODeviceHandle; import org.apache.hyracks.api.util.ExceptionUtils; @@ -30,6 +31,7 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; import org.apache.hyracks.storage.am.lsm.common.api.IoOperationCompleteListener; +import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICachedPage; public abstract class AbstractIoOperation implements ILSMIOOperation { @@ -78,6 +80,27 @@ public abstract class AbstractIoOperation implements ILSMIOOperation { } @Override + public void cleanup(IBufferCache bufferCache) { + LSMComponentFileReferences componentFiles = getComponentFiles(); + if (componentFiles == null) { + return; + } + FileReference[] files = componentFiles.getFileReferences(); + for (FileReference file : files) { + try { + if (file != null) { + bufferCache.closeFileIfOpen(file); + bufferCache.deleteFile(file); + } + } catch (HyracksDataException hde) { + getFailure().addSuppressed(hde); + } + } + } + + protected abstract LSMComponentFileReferences getComponentFiles(); + + @Override public Throwable getFailure() { return failure; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4f66311/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java index d3133ce..cf80472 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java @@ -860,25 +860,6 @@ public abstract class AbstractLSMIndex implements ILSMIndex { } @Override - public void cleanUpFilesForFailedOperation(ILSMIOOperation operation) { - LSMComponentFileReferences componentFiles = operation.getComponentFiles(); - if (componentFiles == null) { - return; - } - FileReference[] files = componentFiles.getFileReferences(); - for (FileReference file : files) { - try { - if (file != null) { - diskBufferCache.closeFileIfOpen(file); - diskBufferCache.deleteFile(file); - } - } catch (Throwable th) { // NOSONAR Must catch all failures - operation.getFailure().addSuppressed(th); - } - } - } - - @Override public String getIndexIdentifier() { return fileManager.getBaseDir().getAbsolutePath(); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4f66311/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java index e9f6f20..dc54e20 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java @@ -553,7 +553,7 @@ public class LSMHarness implements ILSMHarness { } // if the operation failed, we need to cleanup files if (operation.getStatus() == LSMIOOperationStatus.FAILURE) { - lsmIndex.cleanUpFilesForFailedOperation(operation); + operation.cleanup(lsmIndex.getBufferCache()); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4f66311/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexDiskComponentBulkLoader.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexDiskComponentBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexDiskComponentBulkLoader.java index c739ad0..9b36383 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexDiskComponentBulkLoader.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexDiskComponentBulkLoader.java @@ -129,7 +129,7 @@ public class LSMIndexDiskComponentBulkLoader implements IIndexBulkLoader { failed = true; final ILSMIOOperation loadOp = opCtx.getIoOperation(); loadOp.setFailure(th); - lsmIndex.cleanUpFilesForFailedOperation(loadOp); + loadOp.cleanup(lsmIndex.getBufferCache()); } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4f66311/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/NoOpIoOperation.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/NoOpIoOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/NoOpIoOperation.java index 5ee1503..cf0f4e5 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/NoOpIoOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/NoOpIoOperation.java @@ -29,6 +29,7 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; import org.apache.hyracks.storage.am.lsm.common.api.IoOperationCompleteListener; +import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICachedPage; public class NoOpIoOperation implements ILSMIOOperation { @@ -73,7 +74,7 @@ public class NoOpIoOperation implements ILSMIOOperation { } @Override - public LSMComponentFileReferences getComponentFiles() { + public void cleanup(IBufferCache bufferCache) { throw new UnsupportedOperationException(); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4f66311/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java index 3345e3a..a2304ed 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java @@ -29,6 +29,7 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; import org.apache.hyracks.storage.am.lsm.common.api.IoOperationCompleteListener; +import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICachedPage; import org.apache.hyracks.util.trace.ITracer; import org.apache.hyracks.util.trace.ITracer.Scope; @@ -110,8 +111,8 @@ class TracedIOOperation implements ILSMIOOperation { } @Override - public LSMComponentFileReferences getComponentFiles() { - return ioOp.getComponentFiles(); + public void cleanup(IBufferCache bufferCache) { + ioOp.cleanup(bufferCache); } @Override