Murtadha Hubail has submitted this change and it was merged. Change subject: Exclude Temporary Indexes From Replication ......................................................................
Exclude Temporary Indexes From Replication - Exclude non-durable LSM indexes from replication. - Stop heartbeat task after NC has stopped to avoid false failures detection. Change-Id: Icce91a203e04cb068a7a5aa541720bbd0289eacb Reviewed-on: https://asterix-gerrit.ics.uci.edu/777 Tested-by: Jenkins <[email protected]> Reviewed-by: abdullah alamoudi <[email protected]> --- M hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java M hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java M hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java M hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java 4 files changed, 29 insertions(+), 14 deletions(-) Approvals: abdullah alamoudi: Looks good to me, approved Jenkins: Verified diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java index 7b5758c..598d6db 100644 --- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java +++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java @@ -338,12 +338,17 @@ } partitionManager.close(); datasetPartitionManager.close(); - heartbeatTask.cancel(); netManager.stop(); datasetNetworkManager.stop(); queue.stop(); - if (ncAppEntryPoint != null) + if (ncAppEntryPoint != null) { ncAppEntryPoint.stop(); + } + /** + * Stop heartbeat after NC has stopped to avoid false node failure detection + * on CC if an NC takes a long time to stop. + */ + heartbeatTask.cancel(); LOGGER.log(Level.INFO, "Stopped NodeControllerService"); shuttedDown = true; } diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java index c1cef2d..11b933d 100644 --- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java +++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java @@ -39,6 +39,7 @@ public void deactivate(boolean flushOnExit) throws HyracksDataException; + @Override public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback) throws HyracksDataException; @@ -51,4 +52,9 @@ public List<ILSMComponent> getImmutableComponents(); public boolean isPrimaryIndex(); + + /** + * @return true if the index is durable. Otherwise false. + */ + public boolean isDurable(); } diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java index 441dda1..440ad31 100644 --- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java +++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java @@ -33,7 +33,6 @@ import org.apache.hyracks.api.replication.IReplicationJob.ReplicationOperation; import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter; import org.apache.hyracks.storage.am.common.api.ITreeIndex; -import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.ComponentState; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory; @@ -48,8 +47,6 @@ import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache; import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType; import org.apache.hyracks.storage.common.buffercache.IBufferCache; -import org.apache.hyracks.storage.common.buffercache.ICachedPage; -import org.apache.hyracks.storage.common.file.BufferedFileHandle; import org.apache.hyracks.storage.common.file.IFileMapProvider; public abstract class AbstractLSMIndex implements ILSMIndexInternal { @@ -80,11 +77,10 @@ protected boolean memoryComponentsAllocated = false; public AbstractLSMIndex(List<IVirtualBufferCache> virtualBufferCaches, IBufferCache diskBufferCache, - ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider, - double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, - ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, - ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager, - int[] filterFields, boolean durable) { + ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider, double bloomFilterFalsePositiveRate, + ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, + ILSMIOOperationCallback ioOpCallback, ILSMComponentFilterFrameFactory filterFrameFactory, + LSMComponentFilterManager filterManager, int[] filterFields, boolean durable) { this.virtualBufferCaches = virtualBufferCaches; this.diskBufferCache = diskBufferCache; this.diskFileMapProvider = diskFileMapProvider; @@ -149,8 +145,8 @@ } protected void markAsValidInternal(IBufferCache bufferCache, BloomFilter filter) throws HyracksDataException { - if(durable){ - bufferCache.force(filter.getFileId(),true); + if (durable) { + bufferCache.force(filter.getFileId(), true); } } @@ -208,7 +204,7 @@ return diskBufferCache; } - public boolean isEmptyIndex() throws HyracksDataException { + public boolean isEmptyIndex() { boolean isModified = false; for (ILSMComponent c : memoryComponents) { AbstractMemoryLSMComponent mutableComponent = (AbstractMemoryLSMComponent) c; @@ -288,9 +284,16 @@ } } + @Override public abstract void allocateMemoryComponents() throws HyracksDataException; + @Override public boolean isMemoryComponentsAllocated() { return memoryComponentsAllocated; } + + @Override + public boolean isDurable() { + return durable; + } } diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java index 21b0d8a..0224c5c 100644 --- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java +++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java @@ -63,7 +63,8 @@ this.opTracker = opTracker; this.mergePolicy = mergePolicy; fullMergeIsRequested = new AtomicBoolean(); - this.replicationEnabled = replicationEnabled; + //only durable indexes are replicated + this.replicationEnabled = replicationEnabled && lsmIndex.isDurable(); if (replicationEnabled) { this.componentsToBeReplicated = new ArrayList<ILSMComponent>(); } -- To view, visit https://asterix-gerrit.ics.uci.edu/777 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: Icce91a203e04cb068a7a5aa541720bbd0289eacb Gerrit-PatchSet: 3 Gerrit-Project: hyracks Gerrit-Branch: master Gerrit-Owner: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Ian Maxon <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-Reviewer: abdullah alamoudi <[email protected]>
