Michael Blow has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/711
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
As sugggested in ASTERIXDB-1337, this change is to maintain a per-
partition MultitenantVirtualBufferCache budget, as opposed to sharing
the budget across the dataset.
Change-Id: Ie554ab6d9b0d01ed149ab0729d20cdac3c969ec1
---
M
hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
M
hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IVirtualBufferCacheProvider.java
M
hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java
M
hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java
M
hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
M
hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelperFactory.java
M
hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestVirtualBufferCacheProvider.java
7 files changed, 9 insertions(+), 7 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/hyracks refs/changes/11/711/1
diff --git
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
index fd300bd..141b077 100644
---
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
+++
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
@@ -56,7 +56,7 @@
public IndexDataflowHelper
createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext
ctx,
int partition) {
return new LSMBTreeDataflowHelper(opDesc, ctx, partition,
- virtualBufferCacheProvider.getVirtualBufferCaches(ctx),
bloomFilterFalsePositiveRate,
+ virtualBufferCacheProvider.getVirtualBufferCaches(ctx,
opDesc), bloomFilterFalsePositiveRate,
mergePolicyFactory.createMergePolicy(mergePolicyProperties,
ctx), opTrackerFactory,
ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory,
needKeyDupCheck, filterTypeTraits,
filterCmpFactories, btreeFields, filterFields, durable);
diff --git
a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IVirtualBufferCacheProvider.java
b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IVirtualBufferCacheProvider.java
index b5fc198..a53e098 100644
---
a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IVirtualBufferCacheProvider.java
+++
b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IVirtualBufferCacheProvider.java
@@ -22,7 +22,8 @@
import java.util.List;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
public interface IVirtualBufferCacheProvider extends Serializable {
- public List<IVirtualBufferCache>
getVirtualBufferCaches(IHyracksTaskContext ctx);
+ public List<IVirtualBufferCache>
getVirtualBufferCaches(IHyracksTaskContext ctx, IIndexOperatorDescriptor
opDesc);
}
diff --git
a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java
b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java
index 7813ba9..689721a 100644
---
a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java
+++
b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java
@@ -59,7 +59,7 @@
public IndexDataflowHelper
createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext
ctx,
int partition) {
return new LSMInvertedIndexDataflowHelper(opDesc, ctx, partition,
- virtualBufferCacheProvider.getVirtualBufferCaches(ctx),
bloomFilterFalsePositiveRate,
+ virtualBufferCacheProvider.getVirtualBufferCaches(ctx,
opDesc), bloomFilterFalsePositiveRate,
mergePolicyFactory.createMergePolicy(mergePolicyProperties,
ctx), opTrackerFactory,
ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory,
invertedIndexFields, filterTypeTraits,
filterCmpFactories, filterFields,
filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps,
diff --git
a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java
b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java
index 91d461c..7af66ec 100644
---
a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java
+++
b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java
@@ -59,7 +59,7 @@
public IndexDataflowHelper
createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext
ctx,
int partition) {
return new PartitionedLSMInvertedIndexDataflowHelper(opDesc, ctx,
partition,
- virtualBufferCacheProvider.getVirtualBufferCaches(ctx),
bloomFilterFalsePositiveRate,
+ virtualBufferCacheProvider.getVirtualBufferCaches(ctx,
opDesc), bloomFilterFalsePositiveRate,
mergePolicyFactory.createMergePolicy(mergePolicyProperties,
ctx), opTrackerFactory,
ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory,
invertedIndexFields, filterTypeTraits,
filterCmpFactories, filterFields,
filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps,
diff --git
a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
index 4ec5c13..2d2add7 100644
---
a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
+++
b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
@@ -70,7 +70,7 @@
public IndexDataflowHelper
createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext
ctx,
int partition) {
return new LSMRTreeDataflowHelper(opDesc, ctx, partition,
- virtualBufferCacheProvider.getVirtualBufferCaches(ctx),
bloomFilterFalsePositiveRate,
+ virtualBufferCacheProvider.getVirtualBufferCaches(ctx,
opDesc), bloomFilterFalsePositiveRate,
btreeComparatorFactories, valueProviderFactories,
rtreePolicyType,
mergePolicyFactory.createMergePolicy(mergePolicyProperties,
ctx), opTrackerFactory,
ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory,
linearizeCmpFactory, rtreeFields,
diff --git
a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelperFactory.java
b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelperFactory.java
index a8c7896..fe16795 100644
---
a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelperFactory.java
+++
b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelperFactory.java
@@ -67,7 +67,7 @@
public IndexDataflowHelper
createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext
ctx,
int partition) {
return new LSMRTreeWithAntiMatterTuplesDataflowHelper(opDesc, ctx,
partition,
- virtualBufferCacheProvider.getVirtualBufferCaches(ctx),
btreeComparatorFactories,
+ virtualBufferCacheProvider.getVirtualBufferCaches(ctx,
opDesc), btreeComparatorFactories,
valueProviderFactories, rtreePolicyType,
mergePolicyFactory.createMergePolicy(mergePolicyProperties,
ctx), opTrackerFactory,
ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory,
linearizeCmpFactory, rtreeFields, filterTypeTraits,
filterCmpFactories, filterFields, durable);
diff --git
a/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestVirtualBufferCacheProvider.java
b/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestVirtualBufferCacheProvider.java
index 1dba387..5d245b0 100644
---
a/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestVirtualBufferCacheProvider.java
+++
b/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestVirtualBufferCacheProvider.java
@@ -22,6 +22,7 @@
import java.util.List;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import
org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCacheProvider;
import org.apache.hyracks.storage.am.lsm.common.impls.VirtualBufferCache;
@@ -40,7 +41,7 @@
}
@Override
- public List<IVirtualBufferCache>
getVirtualBufferCaches(IHyracksTaskContext ctx) {
+ public List<IVirtualBufferCache>
getVirtualBufferCaches(IHyracksTaskContext ctx, IIndexOperatorDescriptor
opDesc) {
List<IVirtualBufferCache> vbcs = new ArrayList<IVirtualBufferCache>();
for (int i = 0; i < 2; i++) {
IVirtualBufferCache vbc = new VirtualBufferCache(new
HeapBufferAllocator(), pageSize, numPages / 2);
--
To view, visit https://asterix-gerrit.ics.uci.edu/711
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie554ab6d9b0d01ed149ab0729d20cdac3c969ec1
Gerrit-PatchSet: 1
Gerrit-Project: hyracks
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <[email protected]>
