Murtadha Hubail has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/343
Change subject: Add support for persistent local resources to
IndexLifecycleManager
......................................................................
Add support for persistent local resources to IndexLifecycleManager
Change-Id: I2e8e974fc2f746959639ce94351f8e419a7f9093
---
M
hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndexLifecycleManager.java
M
hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IModificationOperationCallbackFactory.java
M
hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
M
hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexLifecycleManager.java
M
hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/NoOpOperationCallbackFactory.java
M
hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
M
hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
M
hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java
8 files changed, 158 insertions(+), 40 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/hyracks refs/changes/43/343/1
diff --git
a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndexLifecycleManager.java
b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndexLifecycleManager.java
index de4ce5a..2f3fdb2 100644
---
a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndexLifecycleManager.java
+++
b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndexLifecycleManager.java
@@ -30,5 +30,18 @@
public void close(long resourceID) throws HyracksDataException;
public List<IIndex> getOpenIndexes();
-
+
+ public boolean supportsPersistentLocalResources();
+
+ public void register(String resourceName, IIndex index) throws
HyracksDataException;
+
+ public void open(String resourceName) throws HyracksDataException;
+
+ public void close(String resourceName) throws HyracksDataException;
+
+ public IIndex getIndex(String resourceName) throws HyracksDataException;
+
+ public void unregister(String resourceName) throws HyracksDataException;
+
+ public IIndex getIndex(int datasetID, long resourceID) throws
HyracksDataException;
}
\ No newline at end of file
diff --git
a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IModificationOperationCallbackFactory.java
b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IModificationOperationCallbackFactory.java
index 968acd0..1d85256 100644
---
a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IModificationOperationCallbackFactory.java
+++
b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IModificationOperationCallbackFactory.java
@@ -20,6 +20,9 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
public interface IModificationOperationCallbackFactory extends Serializable {
- public IModificationOperationCallback
createModificationOperationCallback(long resourceId, Object resource,
IHyracksTaskContext ctx)
- throws HyracksDataException;
+ public IModificationOperationCallback
createModificationOperationCallback(long resourceId, Object resource,
+ IHyracksTaskContext ctx) throws HyracksDataException;
+
+ public IModificationOperationCallback
createModificationOperationCallback(String resourceName, long resourceId,
+ Object resource, IHyracksTaskContext ctx) throws
HyracksDataException;
}
diff --git
a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
index d67f585..816d743 100644
---
a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
+++
b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
@@ -69,11 +69,21 @@
public void create() throws HyracksDataException {
synchronized (lcManager) {
long resourceID = getResourceID();
- index = lcManager.getIndex(resourceID);
- if (index != null) {
- lcManager.unregister(resourceID);
+
+ if (lcManager.supportsPersistentLocalResources()) {
+ index = lcManager.getIndex(file.getFile().getPath());
+ if (index != null) {
+ lcManager.unregister(file.getFile().getPath());
+ } else {
+ index = createIndexInstance();
+ }
} else {
- index = createIndexInstance();
+ index = lcManager.getIndex(resourceID);
+ if (index != null) {
+ lcManager.unregister(resourceID);
+ } else {
+ index = createIndexInstance();
+ }
}
// The previous resource ID needs to be removed since calling
IIndex.create() may possibly destroy
@@ -93,7 +103,12 @@
} catch (IOException e) {
throw new HyracksDataException(e);
}
- lcManager.register(resourceID, index);
+
+ if (lcManager.supportsPersistentLocalResources()) {
+ lcManager.register(file.getFile().getPath(), index);
+ } else {
+ lcManager.register(resourceID, index);
+ }
}
}
@@ -106,19 +121,32 @@
throw new HyracksDataException("Index does not have a valid
resource ID. Has it been created yet?");
}
- index = lcManager.getIndex(resourceID);
- if (index == null) {
- index = createIndexInstance();
- lcManager.register(resourceID, index);
+ if (lcManager.supportsPersistentLocalResources()) {
+ index = lcManager.getIndex(file.getFile().getPath());
+ if (index == null) {
+ index = createIndexInstance();
+ lcManager.register(file.getFile().getPath(), index);
+ }
+ lcManager.open(file.getFile().getPath());
+ } else {
+ index = lcManager.getIndex(resourceID);
+ if (index == null) {
+ index = createIndexInstance();
+ lcManager.register(resourceID, index);
+ }
+ lcManager.open(resourceID);
}
- lcManager.open(resourceID);
}
}
@Override
public void close() throws HyracksDataException {
synchronized (lcManager) {
- lcManager.close(getResourceID());
+ if (lcManager.supportsPersistentLocalResources()) {
+ lcManager.close(file.getFile().getPath());
+ } else {
+ lcManager.close(getResourceID());
+ }
}
}
@@ -126,11 +154,20 @@
public void destroy() throws HyracksDataException {
synchronized (lcManager) {
long resourceID = getResourceID();
- index = lcManager.getIndex(resourceID);
- if (index != null) {
- lcManager.unregister(resourceID);
+ if (lcManager.supportsPersistentLocalResources()) {
+ index = lcManager.getIndex(file.getFile().getPath());
+ if (index != null) {
+ lcManager.unregister(file.getFile().getPath());
+ } else {
+ index = createIndexInstance();
+ }
} else {
- index = createIndexInstance();
+ index = lcManager.getIndex(resourceID);
+ if (index != null) {
+ lcManager.unregister(resourceID);
+ } else {
+ index = createIndexInstance();
+ }
}
if (resourceID != -1) {
diff --git
a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexLifecycleManager.java
b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexLifecycleManager.java
index 8543433..970ef93 100644
---
a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexLifecycleManager.java
+++
b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexLifecycleManager.java
@@ -231,4 +231,39 @@
}
os.write(sb.toString().getBytes());
}
+
+ @Override
+ public void register(String resourceName, IIndex index) throws
HyracksDataException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void open(String resourceName) throws HyracksDataException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean supportsPersistentLocalResources() {
+ return false;
+ }
+
+ @Override
+ public void close(String resourceName) throws HyracksDataException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IIndex getIndex(String resourceName) throws HyracksDataException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void unregister(String resourceName) throws HyracksDataException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IIndex getIndex(int datasetID, long resourceID) throws
HyracksDataException {
+ throw new UnsupportedOperationException();
+ }
}
\ No newline at end of file
diff --git
a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/NoOpOperationCallbackFactory.java
b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/NoOpOperationCallbackFactory.java
index 7a9ae4c..a0f678c 100644
---
a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/NoOpOperationCallbackFactory.java
+++
b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/NoOpOperationCallbackFactory.java
@@ -15,6 +15,7 @@
package edu.uci.ics.hyracks.storage.am.common.impls;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import
edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import
edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallbackFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
@@ -29,7 +30,8 @@
INSTANCE;
@Override
- public IModificationOperationCallback
createModificationOperationCallback(long resourceId, Object resource,
IHyracksTaskContext ctx) {
+ public IModificationOperationCallback
createModificationOperationCallback(long resourceId, Object resource,
+ IHyracksTaskContext ctx) {
return NoOpOperationCallback.INSTANCE;
}
@@ -37,4 +39,10 @@
public ISearchOperationCallback createSearchOperationCallback(long
resourceId, IHyracksTaskContext ctx) {
return NoOpOperationCallback.INSTANCE;
}
+
+ @Override
+ public IModificationOperationCallback
createModificationOperationCallback(String resourceName, long resourceId,
+ Object resource, IHyracksTaskContext ctx) throws
HyracksDataException {
+ return NoOpOperationCallback.INSTANCE;
+ }
}
diff --git
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
index b913e28..87a1acf 100644
---
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
+++
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
@@ -36,8 +36,7 @@
public ExternalBTreeDataflowHelper(IIndexOperatorDescriptor opDesc,
IHyracksTaskContext ctx, int partition,
double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean
needKeyDupCheck, int version,
- boolean durable) {
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean
needKeyDupCheck, int version, boolean durable) {
super(opDesc, ctx, partition, null, bloomFilterFalsePositiveRate,
mergePolicy, opTrackerFactory, ioScheduler,
ioOpCallbackFactory, false, null, null, null, null, durable);
this.version = version;
@@ -46,8 +45,7 @@
public ExternalBTreeDataflowHelper(IIndexOperatorDescriptor opDesc,
IHyracksTaskContext ctx, int partition,
List<IVirtualBufferCache> virtualBufferCaches, ILSMMergePolicy
mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean
needKeyDupCheck, int version,
- boolean durable) {
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean
needKeyDupCheck, int version, boolean durable) {
this(opDesc, ctx, partition, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE,
mergePolicy, opTrackerFactory,
ioScheduler, ioOpCallbackFactory, needKeyDupCheck, version,
durable);
}
@@ -63,10 +61,18 @@
} catch (HyracksDataException e) {
return null;
}
- try {
- index = lcManager.getIndex(resourceID);
- } catch (HyracksDataException e) {
- return null;
+ if (lcManager.supportsPersistentLocalResources()) {
+ try {
+ index = lcManager.getIndex(file.getFile().getPath());
+ } catch (HyracksDataException e) {
+ return null;
+ }
+ } else {
+ try {
+ index = lcManager.getIndex(resourceID);
+ } catch (HyracksDataException e) {
+ return null;
+ }
}
}
return index;
diff --git
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
index ff19c54..075b3ed 100644
---
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
+++
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
@@ -63,10 +63,18 @@
} catch (HyracksDataException e) {
return null;
}
- try {
- index = lcManager.getIndex(resourceID);
- } catch (HyracksDataException e) {
- return null;
+ if (lcManager.supportsPersistentLocalResources()) {
+ try {
+ index = lcManager.getIndex(file.getFile().getPath());
+ } catch (HyracksDataException e) {
+ return null;
+ }
+ } else {
+ try {
+ index = lcManager.getIndex(resourceID);
+ } catch (HyracksDataException e) {
+ return null;
+ }
}
}
return index;
diff --git
a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java
b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java
index 4fee495..252775f 100644
---
a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java
+++
b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java
@@ -77,10 +77,18 @@
} catch (HyracksDataException e) {
return null;
}
- try {
- index = lcManager.getIndex(resourceID);
- } catch (HyracksDataException e) {
- return null;
+ if (lcManager.supportsPersistentLocalResources()) {
+ try {
+ index = lcManager.getIndex(file.getFile().getPath());
+ } catch (HyracksDataException e) {
+ return null;
+ }
+ } else {
+ try {
+ index = lcManager.getIndex(resourceID);
+ } catch (HyracksDataException e) {
+ return null;
+ }
}
}
return index;
@@ -95,11 +103,11 @@
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[]
filterCmpFactories, int[] filterFields)
throws HyracksDataException {
try {
- return LSMRTreeUtils.createExternalRTree(file, diskBufferCache,
diskFileMapProvider, typeTraits,
- rtreeCmpFactories, btreeCmpFactories,
valueProviderFactories, rtreePolicyType,
- bloomFilterFalsePositiveRate, mergePolicy, opTracker,
ioScheduler,
- ioOpCallbackFactory.createIOOperationCallback(),
linearizeCmpFactory, btreeFields, version,
- durable);
+ return LSMRTreeUtils
+ .createExternalRTree(file, diskBufferCache,
diskFileMapProvider, typeTraits, rtreeCmpFactories,
+ btreeCmpFactories, valueProviderFactories,
rtreePolicyType, bloomFilterFalsePositiveRate,
+ mergePolicy, opTracker, ioScheduler,
ioOpCallbackFactory.createIOOperationCallback(),
+ linearizeCmpFactory, btreeFields, version,
durable);
} catch (TreeIndexException e) {
throw new HyracksDataException(e);
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/343
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I2e8e974fc2f746959639ce94351f8e419a7f9093
Gerrit-PatchSet: 1
Gerrit-Project: hyracks
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <[email protected]>
