Change in asterixdb[master]: [NO ISSUE][STO]Ensure Bloom filters are not built for secondary indexes
Luo Chen has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/3366
Change subject: [NO ISSUE][STO]Ensure Bloom filters are not built for secondary
indexes
..
[NO ISSUE][STO]Ensure Bloom filters are not built for secondary indexes
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Calculate bloom filter fields properly based on the index type
so that we do not build unnecessary bloom filters for secondary indexes.
Change-Id: Ib4272e5278bf76fa78af36ccf10d516da12c4810
---
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBTreeResourceFactoryProvider.java
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java
2 files changed, 26 insertions(+), 13 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/66/3366/1
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBTreeResourceFactoryProvider.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBTreeResourceFactoryProvider.java
index 8d37878..4186f4d 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBTreeResourceFactoryProvider.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBTreeResourceFactoryProvider.java
@@ -22,6 +22,7 @@
import java.util.Map;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
+import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider;
import org.apache.asterix.common.context.IStorageComponentProvider;
import org.apache.asterix.external.indexing.FilesIndexDescription;
@@ -155,7 +156,8 @@
}
private static int[] getBloomFilterFields(Dataset dataset, Index index)
throws AlgebricksException {
-if (index.isPrimaryIndex()) {
+// both the Primary index and the Primary Key index have bloom filters
+if (index.isPrimaryIndex() || index.isPrimaryKeyIndex()) {
return dataset.getPrimaryBloomFilterFields();
} else if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
if
(index.getIndexName().equals(IndexingConstants.getFilesIndexName(dataset.getDatasetName(
{
@@ -163,12 +165,17 @@
} else {
return new int[] { index.getKeyFieldNames().size() };
}
+} else if (index.getIndexType() == IndexType.BTREE ||
index.getIndexType() == IndexType.RTREE) {
+// secondary btrees and rtrees do not have bloom filters
+return null;
+} else {
+// inverted indexes has bloom filters on deleted-key btrees
+int numKeys = index.getKeyFieldNames().size();
+int[] bloomFilterKeyFields = new int[numKeys];
+for (int i = 0; i < numKeys; i++) {
+bloomFilterKeyFields[i] = i;
+}
+return bloomFilterKeyFields;
}
-int numKeys = index.getKeyFieldNames().size();
-int[] bloomFilterKeyFields = new int[numKeys];
-for (int i = 0; i < numKeys; i++) {
-bloomFilterKeyFields[i] = i;
-}
-return bloomFilterKeyFields;
}
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java
index 883da84..3f185f1 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java
@@ -22,6 +22,7 @@
import java.util.Map;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
+import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider;
import org.apache.asterix.common.context.IStorageComponentProvider;
import org.apache.asterix.common.exceptions.CompilationException;
@@ -194,12 +195,17 @@
} else {
return new int[] { index.getKeyFieldNames().size() };
}
+} else if (index.getIndexType() == IndexType.BTREE ||
index.getIndexType() == IndexType.RTREE) {
+// secondary btrees and rtrees do not have bloom filters
+return null;
+} else {
+// inverted indexes has bloom filters on deleted-key btrees
+int numKeys = index.getKeyFieldNames().size();
+int[] bloomFilterKeyFields = new int[numKeys];
+for (int i = 0; i < numKeys; i++) {
+bloomFilterKeyFields[i] = i;
+}
+return bloomFilterKeyFields;
Change in asterixdb[master]: [NO ISSUE][STO]Ensure Bloom filters are not built for secondary indexes
Hello Anon. E. Moose (1000171), Jenkins, Murtadha Hubail, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/3366 to look at the new patch set (#2). Change subject: [NO ISSUE][STO]Ensure Bloom filters are not built for secondary indexes .. [NO ISSUE][STO]Ensure Bloom filters are not built for secondary indexes - user model changes: no - storage format changes: no - interface changes: no Details: - Calculate bloom filter fields properly based on the index type so that we do not build unnecessary bloom filters for secondary indexes. Change-Id: Ib4272e5278bf76fa78af36ccf10d516da12c4810 --- M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBTreeResourceFactoryProvider.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java 2 files changed, 26 insertions(+), 13 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/66/3366/2 -- To view, visit https://asterix-gerrit.ics.uci.edu/3366 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-MessageType: newpatchset Gerrit-Change-Id: Ib4272e5278bf76fa78af36ccf10d516da12c4810 Gerrit-Change-Number: 3366 Gerrit-PatchSet: 2 Gerrit-Owner: Luo Chen Gerrit-Reviewer: Anon. E. Moose (1000171) Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Murtadha Hubail
Change in asterixdb[master]: [NO ISSUE][STO]Ensure Bloom filters are not built for secondary indexes
Luo Chen has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/3366 ) Change subject: [NO ISSUE][STO]Ensure Bloom filters are not built for secondary indexes .. Patch Set 2: (1 comment) https://asterix-gerrit.ics.uci.edu/#/c/3366/1/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java File asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java: https://asterix-gerrit.ics.uci.edu/#/c/3366/1/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java@202 PS1, Line 202: hav > have Done -- To view, visit https://asterix-gerrit.ics.uci.edu/3366 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-MessageType: comment Gerrit-Change-Id: Ib4272e5278bf76fa78af36ccf10d516da12c4810 Gerrit-Change-Number: 3366 Gerrit-PatchSet: 2 Gerrit-Owner: Luo Chen Gerrit-Reviewer: Anon. E. Moose (1000171) Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Murtadha Hubail Gerrit-Comment-Date: Sun, 28 Apr 2019 03:14:41 + Gerrit-HasComments: Yes
Change in asterixdb[master]: [NO ISSUE][STO]Ensure Bloom filters are not built for secondary indexes
Luo Chen has submitted this change and it was merged. (
https://asterix-gerrit.ics.uci.edu/3366 )
Change subject: [NO ISSUE][STO]Ensure Bloom filters are not built for secondary
indexes
..
[NO ISSUE][STO]Ensure Bloom filters are not built for secondary indexes
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Calculate bloom filter fields properly based on the index type
so that we do not build unnecessary bloom filters for secondary indexes.
Change-Id: Ib4272e5278bf76fa78af36ccf10d516da12c4810
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3366
Contrib: Jenkins
Tested-by: Jenkins
Integration-Tests: Jenkins
Reviewed-by: Murtadha Hubail
---
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBTreeResourceFactoryProvider.java
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java
2 files changed, 26 insertions(+), 13 deletions(-)
Approvals:
Jenkins: Verified; ; Verified
Murtadha Hubail: Looks good to me, approved
Objections:
Jenkins: Violations found
Anon. E. Moose (1000171): Violations found
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBTreeResourceFactoryProvider.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBTreeResourceFactoryProvider.java
index 8d37878..9a0cd5a 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBTreeResourceFactoryProvider.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBTreeResourceFactoryProvider.java
@@ -22,6 +22,7 @@
import java.util.Map;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
+import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider;
import org.apache.asterix.common.context.IStorageComponentProvider;
import org.apache.asterix.external.indexing.FilesIndexDescription;
@@ -155,7 +156,8 @@
}
private static int[] getBloomFilterFields(Dataset dataset, Index index)
throws AlgebricksException {
-if (index.isPrimaryIndex()) {
+// both the Primary index and the Primary Key index have bloom filters
+if (index.isPrimaryIndex() || index.isPrimaryKeyIndex()) {
return dataset.getPrimaryBloomFilterFields();
} else if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
if
(index.getIndexName().equals(IndexingConstants.getFilesIndexName(dataset.getDatasetName(
{
@@ -163,12 +165,17 @@
} else {
return new int[] { index.getKeyFieldNames().size() };
}
+} else if (index.getIndexType() == IndexType.BTREE ||
index.getIndexType() == IndexType.RTREE) {
+// secondary btrees and rtrees do not have bloom filters
+return null;
+} else {
+// inverted indexes have bloom filters on deleted-key btrees
+int numKeys = index.getKeyFieldNames().size();
+int[] bloomFilterKeyFields = new int[numKeys];
+for (int i = 0; i < numKeys; i++) {
+bloomFilterKeyFields[i] = i;
+}
+return bloomFilterKeyFields;
}
-int numKeys = index.getKeyFieldNames().size();
-int[] bloomFilterKeyFields = new int[numKeys];
-for (int i = 0; i < numKeys; i++) {
-bloomFilterKeyFields[i] = i;
-}
-return bloomFilterKeyFields;
}
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java
index 883da84..21fafaf 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java
@@ -22,6 +22,7 @@
import java.util.Map;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
+import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider;
import org.apache.asterix.common.context.IStorageComponentProvider;
import org.apache.asterix.common.exceptions.CompilationException;
@@ -194,12 +195,17 @@
} else {
return new int[] { index.getKeyFieldNames().size() };
}
+} else if (index.getIndexType() == IndexType.BTREE ||
index.getIndexType() == IndexType.RTREE) {
+// secondary btrees and rtrees do not have bloom filters
+return null;
+} else {
+// inverted indexes have bloom filters on deleted-key btrees
+int numKeys =
Change in asterixdb[master]: [ASTERIXDB-2541] Introduce GreedyScheduler
Hello Anon. E. Moose (1000171), Till Westmann, Jenkins, Michael Blow, Murtadha Hubail, Ian Maxon, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/3284 to look at the new patch set (#5). Change subject: [ASTERIXDB-2541] Introduce GreedyScheduler .. [ASTERIXDB-2541] Introduce GreedyScheduler - user model changes: yes. Add new option: storage.io.scheduler (async/greedy) - storage format changes: no. - interface changes: yes. Introduce IIndexCursorStats Details: - Introduce GreedyScheduler that always executes the merge operation with the smallest number of remaining pages to minimize the number of disk components - Introduce IIndexCursorStats to collect the statistics of index scans. This allows GreedyScheduler to know the remaning pages of merge operations. - Extend AbstractIoOperation so that GreedyScheduler can pause/resume merge operations if needed. Change-Id: I38fe394d1180d4e3f6796064c0e6c6630b6ad303 --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java M asterixdb/asterix-app/src/main/resources/cc.conf M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java M hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksConstants.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTreeRangeSearchCursor.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentScanCursor.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyAbstractCursor.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMergeOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddySearchCursor.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddySortedCursor.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBuddyBTreeMergeCursor.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperationSchedulerFactory.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractAsynchronousScheduler.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AsynchronousScheduler.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ChainedLSMDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/GreedyScheduler.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IoOpera
Change in asterixdb[master]: [ASTERIXDB-2540] Optimize Storage Disk I/O
Hello Anon. E. Moose (1000171), Till Westmann, Jenkins, Michael Blow, Murtadha Hubail, Ian Maxon, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/3285 to look at the new patch set (#7). Change subject: [ASTERIXDB-2540] Optimize Storage Disk I/O .. [ASTERIXDB-2540] Optimize Storage Disk I/O - user model changes: yes. Add two new storage options: storage.disk.force.bytes (default 16MB), storage.write.rate.limit (default 0) - storage format changes: yes. Modified LSM metadata. But this patch is backward-compatible - interface changes: yes. Introduced IPageWriteCallback to LSM indexes Details: - Bypass all queuing (from BufferCache and IOManager) for disk writes. This queuing is unnecessary but destroys fairness among multiple writers. - Introduce IPageWriteCallback to control the behavior of disk page writes. Currently, two features are implemented using this interface, namely regular disk forces and rate limiting. Change-Id: I1f618dc7c186623e860239b4d97640fe3528e75b --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBTreeResourceFactoryProvider.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IRateLimiterFactory.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetRateLimiterFactory.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetResource.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIndexPageWriteCallbackFactory.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/RTreeResourceFactoryProvider.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IPageManager.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/AppendOnlyLinkedMetadataPageManager.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTre
Change in asterixdb[master]: [ASTERIXDB-2540] Optimize Storage Disk I/O
Hello Anon. E. Moose (1000171), Till Westmann, Jenkins, Michael Blow, Murtadha Hubail, Ian Maxon, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/3285 to look at the new patch set (#8). Change subject: [ASTERIXDB-2540] Optimize Storage Disk I/O .. [ASTERIXDB-2540] Optimize Storage Disk I/O - user model changes: yes. Add two new storage options: storage.disk.force.bytes (default 16MB), storage.write.rate.limit (default 0) - storage format changes: yes. Modified LSM metadata. But this patch is backward-compatible - interface changes: yes. Introduced IPageWriteCallback to LSM indexes Details: - Bypass all queuing (from BufferCache and IOManager) for disk writes. This queuing is unnecessary but destroys fairness among multiple writers. - Introduce IPageWriteCallback to control the behavior of disk page writes. Currently, two features are implemented using this interface, namely regular disk forces and rate limiting. Change-Id: I1f618dc7c186623e860239b4d97640fe3528e75b --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBTreeResourceFactoryProvider.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IRateLimiterFactory.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetRateLimiterFactory.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetResource.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIndexPageWriteCallbackFactory.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/RTreeResourceFactoryProvider.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IPageManager.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/AppendOnlyLinkedMetadataPageManager.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTre
Change in asterixdb[master]: [ASTERIXDB-2541] Introduce GreedyScheduler
Luo Chen has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/3284 ) Change subject: [ASTERIXDB-2541] Introduce GreedyScheduler .. Patch Set 6: Code-Review+1 Ready for review -- To view, visit https://asterix-gerrit.ics.uci.edu/3284 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-MessageType: comment Gerrit-Change-Id: I38fe394d1180d4e3f6796064c0e6c6630b6ad303 Gerrit-Change-Number: 3284 Gerrit-PatchSet: 6 Gerrit-Owner: Luo Chen Gerrit-Reviewer: Anon. E. Moose (1000171) Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Michael Blow Gerrit-Reviewer: Murtadha Hubail Gerrit-Reviewer: Till Westmann Gerrit-Comment-Date: Wed, 12 Jun 2019 20:31:20 + Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2540] Optimize Storage Disk I/O
Luo Chen has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/3285 ) Change subject: [ASTERIXDB-2540] Optimize Storage Disk I/O .. Patch Set 8: Code-Review+1 Ready for review -- To view, visit https://asterix-gerrit.ics.uci.edu/3285 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-MessageType: comment Gerrit-Change-Id: I1f618dc7c186623e860239b4d97640fe3528e75b Gerrit-Change-Number: 3285 Gerrit-PatchSet: 8 Gerrit-Owner: Luo Chen Gerrit-Reviewer: Anon. E. Moose (1000171) Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Michael Blow Gerrit-Reviewer: Murtadha Hubail Gerrit-Reviewer: Till Westmann Gerrit-Comment-Date: Wed, 12 Jun 2019 20:31:39 + Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2540] Optimize Storage Disk I/O
Hello Till Westmann, Jenkins, Michael Blow, Murtadha Hubail, Ian Maxon, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/3285 to look at the new patch set (#9). Change subject: [ASTERIXDB-2540] Optimize Storage Disk I/O .. [ASTERIXDB-2540] Optimize Storage Disk I/O - user model changes: yes. Add two new storage options: storage.disk.force.bytes (default 16MB), storage.write.rate.limit (default 0) - storage format changes: yes. Modified LSM metadata. But this patch is backward-compatible - interface changes: yes. Introduced IPageWriteCallback to LSM indexes Details: - Bypass all queuing (from BufferCache and IOManager) for disk writes. This queuing is unnecessary but destroys fairness among multiple writers. - Introduce IPageWriteCallback to control the behavior of disk page writes. Currently, two features are implemented using this interface, namely regular disk forces and rate limiting. Change-Id: I1f618dc7c186623e860239b4d97640fe3528e75b --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBTreeResourceFactoryProvider.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IRateLimiterFactory.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetRateLimiterFactory.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetResource.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIndexPageWriteCallbackFactory.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/RTreeResourceFactoryProvider.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IPageManager.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/AppendOnlyLinkedMetadataPageManager.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack
Change in asterixdb[master]: [ASTERIXDB-2540] Optimize Storage Disk I/O
Luo Chen has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/3285 ) Change subject: [ASTERIXDB-2540] Optimize Storage Disk I/O .. Patch Set 9: (1 comment) https://asterix-gerrit.ics.uci.edu/#/c/3285/8/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java File hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java: https://asterix-gerrit.ics.uci.edu/#/c/3285/8/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java@138 PS8, Line 138: long numElementsHint, boo > Should this return a ILSMDiskComponentBulkLoader instead? Done -- To view, visit https://asterix-gerrit.ics.uci.edu/3285 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-MessageType: comment Gerrit-Change-Id: I1f618dc7c186623e860239b4d97640fe3528e75b Gerrit-Change-Number: 3285 Gerrit-PatchSet: 9 Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Michael Blow Gerrit-Reviewer: Murtadha Hubail Gerrit-Reviewer: Till Westmann Gerrit-Comment-Date: Fri, 14 Jun 2019 16:25:24 + Gerrit-HasComments: Yes
Change in asterixdb[master]: [ASTERIXDB-2540] Optimize Storage Disk I/O
Luo Chen has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/3285 ) Change subject: [ASTERIXDB-2540] Optimize Storage Disk I/O .. Patch Set 9: Code-Review+1 ready for review -- To view, visit https://asterix-gerrit.ics.uci.edu/3285 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-MessageType: comment Gerrit-Change-Id: I1f618dc7c186623e860239b4d97640fe3528e75b Gerrit-Change-Number: 3285 Gerrit-PatchSet: 9 Gerrit-Owner: Luo Chen Gerrit-Reviewer: Anon. E. Moose (1000171) Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Michael Blow Gerrit-Reviewer: Murtadha Hubail Gerrit-Reviewer: Till Westmann Gerrit-Comment-Date: Fri, 14 Jun 2019 17:14:13 + Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2540] Optimize Storage Disk I/O
Luo Chen has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/3285 ) Change subject: [ASTERIXDB-2540] Optimize Storage Disk I/O .. Patch Set 9: -Code-Review (5 comments) https://asterix-gerrit.ics.uci.edu/#/c/3285/9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java File asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java: https://asterix-gerrit.ics.uci.edu/#/c/3285/9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java@99 PS9, Line 99: The maximum disk write rate (bytes/s) for each dataset > what does 0 (the default) mean; we should probably indicate that here Done https://asterix-gerrit.ics.uci.edu/#/c/3285/9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java@101 PS9, Line 101: for > before? Done https://asterix-gerrit.ics.uci.edu/#/c/3285/9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java File asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java: https://asterix-gerrit.ics.uci.edu/#/c/3285/9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java@359 PS9, Line 359: populateRateLimiter > can we return the rate limiter from this method, to avoid the extra lookup? Done https://asterix-gerrit.ics.uci.edu/#/c/3285/9/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/FIFOLocalWriter.java File hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/FIFOLocalWriter.java: https://asterix-gerrit.ics.uci.edu/#/c/3285/9/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/FIFOLocalWriter.java@65 PS9, Line 65: A > a Done https://asterix-gerrit.ics.uci.edu/#/c/3285/9/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/FIFOLocalWriter.java@65 PS9, Line 65: F > f Done -- To view, visit https://asterix-gerrit.ics.uci.edu/3285 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-MessageType: comment Gerrit-Change-Id: I1f618dc7c186623e860239b4d97640fe3528e75b Gerrit-Change-Number: 3285 Gerrit-PatchSet: 9 Gerrit-Owner: Luo Chen Gerrit-Reviewer: Anon. E. Moose (1000171) Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Michael Blow Gerrit-Reviewer: Murtadha Hubail Gerrit-Reviewer: Till Westmann Gerrit-Comment-Date: Tue, 18 Jun 2019 23:59:52 + Gerrit-HasComments: Yes
Change in asterixdb[master]: [ASTERIXDB-2540] Optimize Storage Disk I/O
Hello Anon. E. Moose (1000171), Till Westmann, Jenkins, Michael Blow, Murtadha Hubail, Ian Maxon, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/3285 to look at the new patch set (#10). Change subject: [ASTERIXDB-2540] Optimize Storage Disk I/O .. [ASTERIXDB-2540] Optimize Storage Disk I/O - user model changes: yes. Add two new storage options: storage.disk.force.bytes (default 16MB), storage.write.rate.limit (default 0) - storage format changes: yes. Modified LSM metadata. But this patch is backward-compatible - interface changes: yes. Introduced IPageWriteCallback to LSM indexes Details: - Bypass all queuing (from BufferCache and IOManager) for disk writes. This queuing is unnecessary but destroys fairness among multiple writers. - Introduce IPageWriteCallback to control the behavior of disk page writes. Currently, two features are implemented using this interface, namely regular disk forces and rate limiting. Change-Id: I1f618dc7c186623e860239b4d97640fe3528e75b --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBTreeResourceFactoryProvider.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IRateLimiterFactory.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetRateLimiterFactory.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetResource.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIndexPageWriteCallbackFactory.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/RTreeResourceFactoryProvider.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IPageManager.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/AppendOnlyLinkedMetadataPageManager.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTr
Change in asterixdb[master]: [ASTERIXDB-2540] Optimize Storage Disk I/O
Luo Chen has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/3285 ) Change subject: [ASTERIXDB-2540] Optimize Storage Disk I/O .. Patch Set 10: Code-Review+1 Ready for review -- To view, visit https://asterix-gerrit.ics.uci.edu/3285 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-MessageType: comment Gerrit-Change-Id: I1f618dc7c186623e860239b4d97640fe3528e75b Gerrit-Change-Number: 3285 Gerrit-PatchSet: 10 Gerrit-Owner: Luo Chen Gerrit-Reviewer: Anon. E. Moose (1000171) Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Michael Blow Gerrit-Reviewer: Murtadha Hubail Gerrit-Reviewer: Till Westmann Gerrit-Comment-Date: Wed, 19 Jun 2019 22:53:54 + Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2540] Optimize Storage Disk I/O
Hello Anon. E. Moose (1000171), Till Westmann, Jenkins, Michael Blow, Murtadha Hubail, Ian Maxon, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/3285 to look at the new patch set (#11). Change subject: [ASTERIXDB-2540] Optimize Storage Disk I/O .. [ASTERIXDB-2540] Optimize Storage Disk I/O - user model changes: yes. Add a new storage option: storage.disk.force.bytes (default 16MB), - storage format changes: no. - interface changes: yes. Introduced IPageWriteCallback to LSM indexes Details: - Bypass all queuing (from BufferCache and IOManager) for disk writes. This queuing is unnecessary but destroys fairness among multiple writers. - Introduce IPageWriteCallback to control the behavior of disk page writes. Currently, this interface is used to perform disk forces regularly for each writer thread. Change-Id: I1f618dc7c186623e860239b4d97640fe3528e75b --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBTreeResourceFactoryProvider.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IRateLimiterFactory.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetResource.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIndexPageWriteCallbackFactory.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/RTreeResourceFactoryProvider.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IPageManager.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/AppendOnlyLinkedMetadataPageManager.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache
Change in asterixdb[master]: [ASTERIXDB-2540] Optimize Storage Disk I/O
Hello Anon. E. Moose (1000171), Till Westmann, Jenkins, Michael Blow, Murtadha Hubail, Ian Maxon, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/3285 to look at the new patch set (#12). Change subject: [ASTERIXDB-2540] Optimize Storage Disk I/O .. [ASTERIXDB-2540] Optimize Storage Disk I/O - user model changes: yes. Add a new storage option: storage.disk.force.bytes (default 16MB), - storage format changes: no. - interface changes: yes. Introduced IPageWriteCallback to LSM indexes Details: - Bypass all queuing (from BufferCache and IOManager) for disk writes. This queuing is unnecessary but destroys fairness among multiple writers. - Introduce IPageWriteCallback to control the behavior of disk page writes. Currently, this interface is used to perform disk forces regularly for each writer thread. Change-Id: I1f618dc7c186623e860239b4d97640fe3528e75b --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBTreeResourceFactoryProvider.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IRateLimiterFactory.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetResource.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIndexPageWriteCallbackFactory.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/RTreeResourceFactoryProvider.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IPageManager.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/AppendOnlyLinkedMetadataPageManager.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache
Change in asterixdb[master]: [ASTERIXDB-2540] Optimize Storage Disk I/O
Luo Chen has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/3285 ) Change subject: [ASTERIXDB-2540] Optimize Storage Disk I/O .. Patch Set 12: Code-Review+1 ready for review. -- To view, visit https://asterix-gerrit.ics.uci.edu/3285 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-MessageType: comment Gerrit-Change-Id: I1f618dc7c186623e860239b4d97640fe3528e75b Gerrit-Change-Number: 3285 Gerrit-PatchSet: 12 Gerrit-Owner: Luo Chen Gerrit-Reviewer: Anon. E. Moose (1000171) Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Michael Blow Gerrit-Reviewer: Murtadha Hubail Gerrit-Reviewer: Till Westmann Gerrit-Comment-Date: Mon, 24 Jun 2019 02:09:39 + Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2540] Introduce Disk Write Rate Limiter
Luo Chen has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/3455 Change subject: [ASTERIXDB-2540] Introduce Disk Write Rate Limiter .. [ASTERIXDB-2540] Introduce Disk Write Rate Limiter - user model changes: yes. Add a new storage option: storage.write.rate.limit (default 0) - storage format changes: no. - interface changes: no. Details: - Introduce a disk write rate limiting mechanism to bound the maximum disk write bandwidth usage of large merges. - Disk write limiting is performed for each partition. No change to the storage format. Change-Id: If3cb3df1b3c3b4bbee1ba9ec8ab67c357873ef44 --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBTreeResourceFactoryProvider.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java C asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDiskWriteRateLimiterProvider.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetResource.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DiskWriteRateLimiterProvider.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIndexPageWriteCallbackFactory.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/RTreeResourceFactoryProvider.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IPageManager.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/AppendOnlyLinkedMetadataPageManager.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/m
Change in asterixdb[master]: [ASTERIXDB-2540] Optimize Storage Disk I/O
Hello Anon. E. Moose (1000171), Till Westmann, Jenkins, Michael Blow, Murtadha Hubail, Ian Maxon, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/3285 to look at the new patch set (#13). Change subject: [ASTERIXDB-2540] Optimize Storage Disk I/O .. [ASTERIXDB-2540] Optimize Storage Disk I/O - user model changes: yes. Add a new storage option: storage.disk.force.bytes (default 16MB), - storage format changes: no. - interface changes: yes. Introduced IPageWriteCallback to LSM indexes Details: - Bypass all queuing (from BufferCache and IOManager) for disk writes. This queuing is unnecessary but destroys fairness among multiple writers. - Introduce IPageWriteCallback to control the behavior of disk page writes. Currently, this interface is used to perform disk forces regularly for each writer thread. Change-Id: I1f618dc7c186623e860239b4d97640fe3528e75b --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBTreeResourceFactoryProvider.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIndexPageWriteCallbackFactory.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/RTreeResourceFactoryProvider.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IPageManager.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/AppendOnlyLinkedMetadataPageManager.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBloomFilterDiskComponent.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-
Change in asterixdb[master]: [ASTERIXDB-2540] Optimize Storage Disk I/O
Hello Anon. E. Moose (1000171), Till Westmann, Jenkins, Michael Blow, Murtadha Hubail, Ian Maxon, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/3285 to look at the new patch set (#14). Change subject: [ASTERIXDB-2540] Optimize Storage Disk I/O .. [ASTERIXDB-2540] Optimize Storage Disk I/O - user model changes: yes. Add a new storage option: storage.disk.force.bytes (default 16MB), - storage format changes: no. - interface changes: yes. Introduced IPageWriteCallback to LSM indexes Details: - Bypass all queuing (from BufferCache and IOManager) for disk writes. This queuing is unnecessary but destroys fairness among multiple writers. - Introduce IPageWriteCallback to control the behavior of disk page writes. Currently, this interface is used to perform disk forces regularly for each writer thread. Change-Id: I1f618dc7c186623e860239b4d97640fe3528e75b --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBTreeResourceFactoryProvider.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIndexPageWriteCallbackFactory.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/RTreeResourceFactoryProvider.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IPageManager.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/AppendOnlyLinkedMetadataPageManager.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResourceFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBloomFilterDiskComponent.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-
Change in asterixdb[master]: [ASTERIXDB-2540] Optimize Storage Disk I/O
Luo Chen has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/3285 ) Change subject: [ASTERIXDB-2540] Optimize Storage Disk I/O .. Patch Set 15: Code-Review+1 Ready for review. -- To view, visit https://asterix-gerrit.ics.uci.edu/3285 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-MessageType: comment Gerrit-Change-Id: I1f618dc7c186623e860239b4d97640fe3528e75b Gerrit-Change-Number: 3285 Gerrit-PatchSet: 15 Gerrit-Owner: Luo Chen Gerrit-Reviewer: Anon. E. Moose (1000171) Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Michael Blow Gerrit-Reviewer: Murtadha Hubail Gerrit-Reviewer: Till Westmann Gerrit-Comment-Date: Wed, 26 Jun 2019 05:06:36 + Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2600] Introduce ConcurrentMergePolicy
Hello Anon. E. Moose (1000171), Till Westmann, Jenkins, Michael Blow, Murtadha Hubail, Ian Maxon, Dmitry Lychagin, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/3463 to look at the new patch set (#2). Change subject: [ASTERIXDB-2600] Introduce ConcurrentMergePolicy .. [ASTERIXDB-2600] Introduce ConcurrentMergePolicy - user model changes: yes. Add a new merge policy and make it as default - storage format changes: no. - interface changes: no. Details: - Introduce ConcurrentMergePolicy that performs concurrent merges without the maximum component size. - Make this merge policy as the default merge policy in AsterixDB since the PrefixMergePolicy has made some wrong design decisions. Change-Id: I2ed79847584b9fe846d62ad56ee094863538a2a2 --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-concurrent-merge-policy/using-concurrent-merge-policy.1.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-concurrent-merge-policy/using-concurrent-merge-policy.2.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-concurrent-merge-policy/using-concurrent-merge-policy.3.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-concurrent-merge-policy/using-concurrent-merge-policy.4.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-concurrent-merge-policy/using-concurrent-merge-policy.5.query.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/results/dml/using-concurrent-merge-policy/using-concurrent-merge-policy.1.adm M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ConcurrentMergePolicy.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ConcurrentMergePolicyFactory.java A hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/ConcurrentMergePolicyTest.java 14 files changed, 796 insertions(+), 8 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/63/3463/2 -- To view, visit https://asterix-gerrit.ics.uci.edu/3463 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-MessageType: newpatchset Gerrit-Change-Id: I2ed79847584b9fe846d62ad56ee094863538a2a2 Gerrit-Change-Number: 3463 Gerrit-PatchSet: 2 Gerrit-Owner: Luo Chen Gerrit-Reviewer: Anon. E. Moose (1000171) Gerrit-Reviewer: Dmitry Lychagin Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Michael Blow Gerrit-Reviewer: Murtadha Hubail Gerrit-Reviewer: Till Westmann
Change in asterixdb[master]: [ASTERIXDB-2600] Introduce ConcurrentMergePolicy
Hello Anon. E. Moose (1000171), Till Westmann, Jenkins, Michael Blow, Murtadha Hubail, Ian Maxon, Dmitry Lychagin, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/3463 to look at the new patch set (#3). Change subject: [ASTERIXDB-2600] Introduce ConcurrentMergePolicy .. [ASTERIXDB-2600] Introduce ConcurrentMergePolicy - user model changes: yes. Add a new merge policy and make it as default - storage format changes: no. - interface changes: no. Details: - Introduce ConcurrentMergePolicy that performs concurrent merges without the maximum component size. - Make this merge policy as the default merge policy in AsterixDB since the PrefixMergePolicy has made some wrong design decisions. Change-Id: I2ed79847584b9fe846d62ad56ee094863538a2a2 --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-concurrent-merge-policy/using-concurrent-merge-policy.1.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-concurrent-merge-policy/using-concurrent-merge-policy.2.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-concurrent-merge-policy/using-concurrent-merge-policy.3.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-concurrent-merge-policy/using-concurrent-merge-policy.4.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-concurrent-merge-policy/using-concurrent-merge-policy.5.query.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv02/cross-dv02.1.adm M asterixdb/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv04/cross-dv04.1.adm M asterixdb/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv19/cross-dv19.1.adm A asterixdb/asterix-app/src/test/resources/runtimets/results/dml/using-concurrent-merge-policy/using-concurrent-merge-policy.1.adm M asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/udf23/udf23.1.adm M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/DatasetDeclParametersUtil.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ConcurrentMergePolicy.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ConcurrentMergePolicyFactory.java A hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/ConcurrentMergePolicyTest.java 19 files changed, 826 insertions(+), 31 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/63/3463/3 -- To view, visit https://asterix-gerrit.ics.uci.edu/3463 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-MessageType: newpatchset Gerrit-Change-Id: I2ed79847584b9fe846d62ad56ee094863538a2a2 Gerrit-Change-Number: 3463 Gerrit-PatchSet: 3 Gerrit-Owner: Luo Chen Gerrit-Reviewer: Anon. E. Moose (1000171) Gerrit-Reviewer: Dmitry Lychagin Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Michael Blow Gerrit-Reviewer: Murtadha Hubail Gerrit-Reviewer: Till Westmann
Change in asterixdb[master]: [ASTERIXDB-2600] Introduce ConcurrentMergePolicy
Hello Anon. E. Moose (1000171), Till Westmann, Jenkins, Michael Blow, Murtadha Hubail, Ian Maxon, Dmitry Lychagin, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/3463 to look at the new patch set (#4). Change subject: [ASTERIXDB-2600] Introduce ConcurrentMergePolicy .. [ASTERIXDB-2600] Introduce ConcurrentMergePolicy - user model changes: yes. Add a new merge policy and make it as default - storage format changes: no. - interface changes: no. Details: - Introduce ConcurrentMergePolicy that performs concurrent merges without the maximum component size. - Make this merge policy as the default merge policy in AsterixDB since the PrefixMergePolicy has made some wrong design decisions. Change-Id: I2ed79847584b9fe846d62ad56ee094863538a2a2 --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataTxnTest.java A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-concurrent-merge-policy/using-concurrent-merge-policy.1.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-concurrent-merge-policy/using-concurrent-merge-policy.2.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-concurrent-merge-policy/using-concurrent-merge-policy.3.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-concurrent-merge-policy/using-concurrent-merge-policy.4.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-concurrent-merge-policy/using-concurrent-merge-policy.5.query.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv02/cross-dv02.1.adm M asterixdb/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv04/cross-dv04.1.adm M asterixdb/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv19/cross-dv19.1.adm A asterixdb/asterix-app/src/test/resources/runtimets/results/dml/using-concurrent-merge-policy/using-concurrent-merge-policy.1.adm M asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/udf23/udf23.1.adm M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/DatasetDeclParametersUtil.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ConcurrentMergePolicy.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ConcurrentMergePolicyFactory.java A hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/ConcurrentMergePolicyTest.java 20 files changed, 827 insertions(+), 32 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/63/3463/4 -- To view, visit https://asterix-gerrit.ics.uci.edu/3463 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-MessageType: newpatchset Gerrit-Change-Id: I2ed79847584b9fe846d62ad56ee094863538a2a2 Gerrit-Change-Number: 3463 Gerrit-PatchSet: 4 Gerrit-Owner: Luo Chen Gerrit-Reviewer: Anon. E. Moose (1000171) Gerrit-Reviewer: Dmitry Lychagin Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Michael Blow Gerrit-Reviewer: Murtadha Hubail Gerrit-Reviewer: Till Westmann
Change in asterixdb[master]: Carry filter in 2ndary-to-primary index search
Luo Chen has posted comments on this change.
Change subject: Carry filter in 2ndary-to-primary index search
..
Patch Set 6:
(3 comments)
https://asterix-gerrit.ics.uci.edu/#/c/1720/6/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
File
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java:
PS6, Line 358: // TODO look at here about the variables
Is this TODO necessary to this patch? Or just a TODO marker on local branch?
https://asterix-gerrit.ics.uci.edu/#/c/1720/6/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndexDataflowHelper.java
File
hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndexDataflowHelper.java:
PS6, Line 44: public int getNumFilterFields();
Is this method used for supporting multiple filters on an index?
https://asterix-gerrit.ics.uci.edu/#/c/1720/6/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
File
hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java:
PS6, Line 266: int[] offsets = nonFilterTupleBuild.getFieldEndOffsets();
: for (int i = 0; i < offsets.length; i++) {
: int start = i > 0 ? offsets[i - 1] : 0;
: tb.addField(nonFilterTupleBuild.getByteArray(),
start, offsets[i]);
: }
Why would filter tuple == null?
If a dataset does not have filter, could we simply leave out the filter in the
buffer (instead of writing a dummy filter tule)?
--
To view, visit https://asterix-gerrit.ics.uci.edu/1720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I287f1dbd230aa649f1350114abf0a1d47e2bb53c
Gerrit-PatchSet: 6
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Jianfeng Jia
Gerrit-Reviewer: Ian Maxon
Gerrit-Reviewer: Jenkins
Gerrit-Reviewer: Jianfeng Jia
Gerrit-Reviewer: Luo Chen
Gerrit-Reviewer: Yingyi Bu
Gerrit-HasComments: Yes
Change in asterixdb[master]: Carry filter in 2ndary-to-primary index search
Luo Chen has posted comments on this change. Change subject: Carry filter in 2ndary-to-primary index search .. Patch Set 8: Code-Review+1 -- To view, visit https://asterix-gerrit.ics.uci.edu/1720 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I287f1dbd230aa649f1350114abf0a1d47e2bb53c Gerrit-PatchSet: 8 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Jianfeng Jia Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Yingyi Bu Gerrit-HasComments: No
Change in asterixdb[master]: Implemented LSM disk components alignment for datasets using...
Luo Chen has uploaded a new patch set (#2). Change subject: Implemented LSM disk components alignment for datasets using CorrelatedPrefixMergePolicy. The basic idea is to always flush/merge all indexes of a correlated dataset together into a group. Design doc:https://cwiki.apache.org/confluence/display/ASTERIXDB/L .. Implemented LSM disk components alignment for datasets using CorrelatedPrefixMergePolicy. The basic idea is to always flush/merge all indexes of a correlated dataset together into a group. Design doc:https://cwiki.apache.org/confluence/display/ASTERIXDB/LSM+Component+Alignment+Design+Doc The following commits from your working branch will be included: commit a52aa36053fae966ab8246aefee2f5d98f484350 Author: luochen01 Date: Wed May 3 17:36:32 2017 -0700 fix compilation error commit 6e2cbe84db715da2cd039f721456d56414c15847 Author: luochen01 Date: Wed May 3 12:10:17 2017 -0700 clean up error messages commit 997bb839229f43bcbfab6d946e8aaa6e024c31db Author: luochen01 Date: Wed May 3 11:33:32 2017 -0700 Prepare to commit. commit 0235ac99b9459b2ab845d1dd819cc8a102192414 Author: Till Westmann Date: Sat Apr 29 15:56:32 2017 -0700 Add support for binary test results Reviewed-on: https://asterix-gerrit.ics.uci.edu/1710 Sonar-Qube: Jenkins Tested-by: Jenkins BAD: Jenkins Integration-Tests: Jenkins Reviewed-by: Till Westmann commit f1bda46e5588d4b950f4ca231345a7bda651f6f4 Author: Abdullah Alamoudi Date: Sat Apr 29 07:03:57 2017 +0300 Make Index Op Contexts fields private Reviewed-on: https://asterix-gerrit.ics.uci.edu/1712 Tested-by: Jenkins BAD: Jenkins Integration-Tests: Jenkins Reviewed-by: abdullah alamoudi commit 0d051192b8bab1f5f69b6eb5673d038216d13588 Author: Michael Blow Date: Fri Apr 28 19:24:18 2017 -0400 Remove Unused / Historical Hyracks Modules Reviewed-on: https://asterix-gerrit.ics.uci.edu/1714 Sonar-Qube: Jenkins Tested-by: Jenkins BAD: Jenkins Integration-Tests: Jenkins Reviewed-by: Ian Maxon commit 5451aa9f28be0ac080f9fb341d3c799814a3308e Author: Michael Blow Date: Fri Apr 28 17:08:45 2017 -0400 Ensure Proper Process is Killed With Managix Stop Reviewed-on: https://asterix-gerrit.ics.uci.edu/1707 Reviewed-by: Ian Maxon Sonar-Qube: Jenkins Tested-by: Jenkins commit ad424448a387f1b893ca73a0790d16dbd2b09651 Author: Michael Blow Date: Thu Apr 27 16:56:39 2017 -0400 Cleanup Logging, Report Joined Nodes, Misc Cleanup - Minor refactoring of NodeControllerService startup - Cleanup logging in GlobalRecoveryManager / LifeCycleComponentManager - Enable TestExecutor to accept non-200 status codes - Use ExecutorService for GlobalRecovery thread - Eliminate NPE when metadata node goes down before global recovery starts Reviewed-on: https://asterix-gerrit.ics.uci.edu/1706 Tested-by: Jenkins BAD: Jenkins Reviewed-by: Till Westmann Integration-Tests: Jenkins commit 013426cd4ec934623d1e27fd44ba1e63b8aae0a7 Author: Till Westmann Date: Tue Apr 25 23:17:04 2017 -0700 Document the Query Service API Reviewed-on: https://asterix-gerrit.ics.uci.edu/1698 Sonar-Qube: Jenkins Tested-by: Jenkins BAD: Jenkins Integration-Tests: Jenkins Reviewed-by: Till Westmann Reviewed-by: Michael Carey commit 5f4a743e55449cc0f1c0a48e3e8afd24dbe761cc Author: Xikui Wang Date: Tue Apr 25 13:31:16 2017 -0700 Fix NULL/MISSING field value handing in Java UDF Add NULL/MISSING value support in Java UDF. This fixed the NPE when there is a NULL/MISSING field in a record. Reviewed-on: https://asterix-gerrit.ics.uci.edu/1695 Sonar-Qube: Jenkins Tested-by: Jenkins BAD: Jenkins Integration-Tests: Jenkins Reviewed-by: Yingyi Bu commit 37dba5e4d45ce3c86e7a797e0b1bcb101231e4b2 Author: Till Westmann Date: Mon Apr 24 13:43:38 2017 -0700 Remove IOMVisitor and its only (unused) implementation Reviewed-on: https://asterix-gerrit.ics.uci.edu/1699 Sonar-Qube: Jenkins Tested-by: Jenkins Integration-Tests: Jenkins Reviewed-by: Yingyi Bu commit 5449e85ab32d51d88e29b1f50d8450d1bd7a05bd Author: Taewoo Kim Date: Sun Apr 23 17:29:52 2017 -0700 Index-Only Plan Patch Set 4: Index SearchCursor adjustment - Modify the search cursors to maintain the result of proceed() since the result will be required to check whether the given tuple is qualified to be part of an index-only plan result or not. More details will be followed in the next patch set. - Fix the search cursors to call cancel() correctly. Reviewed-on: https://asterix-gerrit.ics.uci.edu/1630 Tested-by: Jenkins BAD: Jenkins Integration-Tests: Jenkins Reviewed-by: Yingyi Bu commit 53ee561c1d10db8d612240df2ac4bb3478925cfe
Change in asterixdb[master]: Implemented LSM disk components alignment for datasets using...
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1725 to look at the new patch set (#3). Change subject: Implemented LSM disk components alignment for datasets using CorrelatedPrefixMergePolicy. The basic idea is to always flush/merge all indexes of a correlated dataset together into a group. Design doc:https://cwiki.apache.org/confluence/display/ASTERIXDB/L .. Implemented LSM disk components alignment for datasets using CorrelatedPrefixMergePolicy. The basic idea is to always flush/merge all indexes of a correlated dataset together into a group. Design doc:https://cwiki.apache.org/confluence/display/ASTERIXDB/LSM+Component+Alignment+Design+Doc The following commits from your working branch will be included: commit a52aa36053fae966ab8246aefee2f5d98f484350 Author: luochen01 Date: Wed May 3 17:36:32 2017 -0700 fix compilation error commit 6e2cbe84db715da2cd039f721456d56414c15847 Author: luochen01 Date: Wed May 3 12:10:17 2017 -0700 clean up error messages commit 997bb839229f43bcbfab6d946e8aaa6e024c31db Author: luochen01 Date: Wed May 3 11:33:32 2017 -0700 Prepare to commit. commit 0235ac99b9459b2ab845d1dd819cc8a102192414 Author: Till Westmann Date: Sat Apr 29 15:56:32 2017 -0700 Add support for binary test results Reviewed-on: https://asterix-gerrit.ics.uci.edu/1710 Sonar-Qube: Jenkins Tested-by: Jenkins BAD: Jenkins Integration-Tests: Jenkins Reviewed-by: Till Westmann commit f1bda46e5588d4b950f4ca231345a7bda651f6f4 Author: Abdullah Alamoudi Date: Sat Apr 29 07:03:57 2017 +0300 Make Index Op Contexts fields private Reviewed-on: https://asterix-gerrit.ics.uci.edu/1712 Tested-by: Jenkins BAD: Jenkins Integration-Tests: Jenkins Reviewed-by: abdullah alamoudi commit 0d051192b8bab1f5f69b6eb5673d038216d13588 Author: Michael Blow Date: Fri Apr 28 19:24:18 2017 -0400 Remove Unused / Historical Hyracks Modules Reviewed-on: https://asterix-gerrit.ics.uci.edu/1714 Sonar-Qube: Jenkins Tested-by: Jenkins BAD: Jenkins Integration-Tests: Jenkins Reviewed-by: Ian Maxon commit 5451aa9f28be0ac080f9fb341d3c799814a3308e Author: Michael Blow Date: Fri Apr 28 17:08:45 2017 -0400 Ensure Proper Process is Killed With Managix Stop Reviewed-on: https://asterix-gerrit.ics.uci.edu/1707 Reviewed-by: Ian Maxon Sonar-Qube: Jenkins Tested-by: Jenkins commit ad424448a387f1b893ca73a0790d16dbd2b09651 Author: Michael Blow Date: Thu Apr 27 16:56:39 2017 -0400 Cleanup Logging, Report Joined Nodes, Misc Cleanup - Minor refactoring of NodeControllerService startup - Cleanup logging in GlobalRecoveryManager / LifeCycleComponentManager - Enable TestExecutor to accept non-200 status codes - Use ExecutorService for GlobalRecovery thread - Eliminate NPE when metadata node goes down before global recovery starts Reviewed-on: https://asterix-gerrit.ics.uci.edu/1706 Tested-by: Jenkins BAD: Jenkins Reviewed-by: Till Westmann Integration-Tests: Jenkins commit 013426cd4ec934623d1e27fd44ba1e63b8aae0a7 Author: Till Westmann Date: Tue Apr 25 23:17:04 2017 -0700 Document the Query Service API Reviewed-on: https://asterix-gerrit.ics.uci.edu/1698 Sonar-Qube: Jenkins Tested-by: Jenkins BAD: Jenkins Integration-Tests: Jenkins Reviewed-by: Till Westmann Reviewed-by: Michael Carey commit 5f4a743e55449cc0f1c0a48e3e8afd24dbe761cc Author: Xikui Wang Date: Tue Apr 25 13:31:16 2017 -0700 Fix NULL/MISSING field value handing in Java UDF Add NULL/MISSING value support in Java UDF. This fixed the NPE when there is a NULL/MISSING field in a record. Reviewed-on: https://asterix-gerrit.ics.uci.edu/1695 Sonar-Qube: Jenkins Tested-by: Jenkins BAD: Jenkins Integration-Tests: Jenkins Reviewed-by: Yingyi Bu commit 37dba5e4d45ce3c86e7a797e0b1bcb101231e4b2 Author: Till Westmann Date: Mon Apr 24 13:43:38 2017 -0700 Remove IOMVisitor and its only (unused) implementation Reviewed-on: https://asterix-gerrit.ics.uci.edu/1699 Sonar-Qube: Jenkins Tested-by: Jenkins Integration-Tests: Jenkins Reviewed-by: Yingyi Bu commit 5449e85ab32d51d88e29b1f50d8450d1bd7a05bd Author: Taewoo Kim Date: Sun Apr 23 17:29:52 2017 -0700 Index-Only Plan Patch Set 4: Index SearchCursor adjustment - Modify the search cursors to maintain the result of proceed() since the result will be required to check whether the given tuple is qualified to be part of an index-only plan result or not. More details will be followed in the next patch set. - Fix the search cursors to call cancel() correctly. Reviewed-on: https://asterix-gerrit.ics.uci.edu/1630 Tested-by: Jenkins BAD: Jenkins In
Change in asterixdb[master]: Implemented LSM disk components alignment for datasets using...
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1725 to look at the new patch set (#4). Change subject: Implemented LSM disk components alignment for datasets using CorrelatedPrefixMergePolicy. The basic idea is to always flush/merge all indexes of a correlated dataset together into a group. Design doc:https://cwiki.apache.org/confluence/display/ASTERIXDB/L .. Implemented LSM disk components alignment for datasets using CorrelatedPrefixMergePolicy. The basic idea is to always flush/merge all indexes of a correlated dataset together into a group. Design doc:https://cwiki.apache.org/confluence/display/ASTERIXDB/LSM+Component+Alignment+Design+Doc The following commits from your working branch will be included: commit a52aa36053fae966ab8246aefee2f5d98f484350 Author: luochen01 Date: Wed May 3 17:36:32 2017 -0700 fix compilation error commit 6e2cbe84db715da2cd039f721456d56414c15847 Author: luochen01 Date: Wed May 3 12:10:17 2017 -0700 clean up error messages commit 997bb839229f43bcbfab6d946e8aaa6e024c31db Author: luochen01 Date: Wed May 3 11:33:32 2017 -0700 Prepare to commit. commit 0235ac99b9459b2ab845d1dd819cc8a102192414 Author: Till Westmann Date: Sat Apr 29 15:56:32 2017 -0700 Add support for binary test results Reviewed-on: https://asterix-gerrit.ics.uci.edu/1710 Sonar-Qube: Jenkins Tested-by: Jenkins BAD: Jenkins Integration-Tests: Jenkins Reviewed-by: Till Westmann commit f1bda46e5588d4b950f4ca231345a7bda651f6f4 Author: Abdullah Alamoudi Date: Sat Apr 29 07:03:57 2017 +0300 Make Index Op Contexts fields private Reviewed-on: https://asterix-gerrit.ics.uci.edu/1712 Tested-by: Jenkins BAD: Jenkins Integration-Tests: Jenkins Reviewed-by: abdullah alamoudi commit 0d051192b8bab1f5f69b6eb5673d038216d13588 Author: Michael Blow Date: Fri Apr 28 19:24:18 2017 -0400 Remove Unused / Historical Hyracks Modules Reviewed-on: https://asterix-gerrit.ics.uci.edu/1714 Sonar-Qube: Jenkins Tested-by: Jenkins BAD: Jenkins Integration-Tests: Jenkins Reviewed-by: Ian Maxon commit 5451aa9f28be0ac080f9fb341d3c799814a3308e Author: Michael Blow Date: Fri Apr 28 17:08:45 2017 -0400 Ensure Proper Process is Killed With Managix Stop Reviewed-on: https://asterix-gerrit.ics.uci.edu/1707 Reviewed-by: Ian Maxon Sonar-Qube: Jenkins Tested-by: Jenkins commit ad424448a387f1b893ca73a0790d16dbd2b09651 Author: Michael Blow Date: Thu Apr 27 16:56:39 2017 -0400 Cleanup Logging, Report Joined Nodes, Misc Cleanup - Minor refactoring of NodeControllerService startup - Cleanup logging in GlobalRecoveryManager / LifeCycleComponentManager - Enable TestExecutor to accept non-200 status codes - Use ExecutorService for GlobalRecovery thread - Eliminate NPE when metadata node goes down before global recovery starts Reviewed-on: https://asterix-gerrit.ics.uci.edu/1706 Tested-by: Jenkins BAD: Jenkins Reviewed-by: Till Westmann Integration-Tests: Jenkins commit 013426cd4ec934623d1e27fd44ba1e63b8aae0a7 Author: Till Westmann Date: Tue Apr 25 23:17:04 2017 -0700 Document the Query Service API Reviewed-on: https://asterix-gerrit.ics.uci.edu/1698 Sonar-Qube: Jenkins Tested-by: Jenkins BAD: Jenkins Integration-Tests: Jenkins Reviewed-by: Till Westmann Reviewed-by: Michael Carey commit 5f4a743e55449cc0f1c0a48e3e8afd24dbe761cc Author: Xikui Wang Date: Tue Apr 25 13:31:16 2017 -0700 Fix NULL/MISSING field value handing in Java UDF Add NULL/MISSING value support in Java UDF. This fixed the NPE when there is a NULL/MISSING field in a record. Reviewed-on: https://asterix-gerrit.ics.uci.edu/1695 Sonar-Qube: Jenkins Tested-by: Jenkins BAD: Jenkins Integration-Tests: Jenkins Reviewed-by: Yingyi Bu commit 37dba5e4d45ce3c86e7a797e0b1bcb101231e4b2 Author: Till Westmann Date: Mon Apr 24 13:43:38 2017 -0700 Remove IOMVisitor and its only (unused) implementation Reviewed-on: https://asterix-gerrit.ics.uci.edu/1699 Sonar-Qube: Jenkins Tested-by: Jenkins Integration-Tests: Jenkins Reviewed-by: Yingyi Bu commit 5449e85ab32d51d88e29b1f50d8450d1bd7a05bd Author: Taewoo Kim Date: Sun Apr 23 17:29:52 2017 -0700 Index-Only Plan Patch Set 4: Index SearchCursor adjustment - Modify the search cursors to maintain the result of proceed() since the result will be required to check whether the given tuple is qualified to be part of an index-only plan result or not. More details will be followed in the next patch set. - Fix the search cursors to call cancel() correctly. Reviewed-on: https://asterix-gerrit.ics.uci.edu/1630 Tested-by: Jenkins BAD: Jenkins In
Change in asterixdb[master]: Implemented LSM disk components alignment for datasets using...
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1725 to look at the new patch set (#5). Change subject: Implemented LSM disk components alignment for datasets using CorrelatedPrefixMergePolicy. The basic idea is to always flush/merge all indexes of a correlated dataset together into a group. Design doc:https://cwiki.apache.org/confluence/display/ASTERIXDB/L .. Implemented LSM disk components alignment for datasets using CorrelatedPrefixMergePolicy. The basic idea is to always flush/merge all indexes of a correlated dataset together into a group. Design doc:https://cwiki.apache.org/confluence/display/ASTERIXDB/LSM+Component+Alignment+Design+Doc The following commits from your working branch will be included: commit a52aa36053fae966ab8246aefee2f5d98f484350 Author: luochen01 Date: Wed May 3 17:36:32 2017 -0700 fix compilation error commit 6e2cbe84db715da2cd039f721456d56414c15847 Author: luochen01 Date: Wed May 3 12:10:17 2017 -0700 clean up error messages commit 997bb839229f43bcbfab6d946e8aaa6e024c31db Author: luochen01 Date: Wed May 3 11:33:32 2017 -0700 Prepare to commit. commit 0235ac99b9459b2ab845d1dd819cc8a102192414 Author: Till Westmann Date: Sat Apr 29 15:56:32 2017 -0700 Add support for binary test results Reviewed-on: https://asterix-gerrit.ics.uci.edu/1710 Sonar-Qube: Jenkins Tested-by: Jenkins BAD: Jenkins Integration-Tests: Jenkins Reviewed-by: Till Westmann commit f1bda46e5588d4b950f4ca231345a7bda651f6f4 Author: Abdullah Alamoudi Date: Sat Apr 29 07:03:57 2017 +0300 Make Index Op Contexts fields private Reviewed-on: https://asterix-gerrit.ics.uci.edu/1712 Tested-by: Jenkins BAD: Jenkins Integration-Tests: Jenkins Reviewed-by: abdullah alamoudi commit 0d051192b8bab1f5f69b6eb5673d038216d13588 Author: Michael Blow Date: Fri Apr 28 19:24:18 2017 -0400 Remove Unused / Historical Hyracks Modules Reviewed-on: https://asterix-gerrit.ics.uci.edu/1714 Sonar-Qube: Jenkins Tested-by: Jenkins BAD: Jenkins Integration-Tests: Jenkins Reviewed-by: Ian Maxon commit 5451aa9f28be0ac080f9fb341d3c799814a3308e Author: Michael Blow Date: Fri Apr 28 17:08:45 2017 -0400 Ensure Proper Process is Killed With Managix Stop Reviewed-on: https://asterix-gerrit.ics.uci.edu/1707 Reviewed-by: Ian Maxon Sonar-Qube: Jenkins Tested-by: Jenkins commit ad424448a387f1b893ca73a0790d16dbd2b09651 Author: Michael Blow Date: Thu Apr 27 16:56:39 2017 -0400 Cleanup Logging, Report Joined Nodes, Misc Cleanup - Minor refactoring of NodeControllerService startup - Cleanup logging in GlobalRecoveryManager / LifeCycleComponentManager - Enable TestExecutor to accept non-200 status codes - Use ExecutorService for GlobalRecovery thread - Eliminate NPE when metadata node goes down before global recovery starts Reviewed-on: https://asterix-gerrit.ics.uci.edu/1706 Tested-by: Jenkins BAD: Jenkins Reviewed-by: Till Westmann Integration-Tests: Jenkins commit 013426cd4ec934623d1e27fd44ba1e63b8aae0a7 Author: Till Westmann Date: Tue Apr 25 23:17:04 2017 -0700 Document the Query Service API Reviewed-on: https://asterix-gerrit.ics.uci.edu/1698 Sonar-Qube: Jenkins Tested-by: Jenkins BAD: Jenkins Integration-Tests: Jenkins Reviewed-by: Till Westmann Reviewed-by: Michael Carey commit 5f4a743e55449cc0f1c0a48e3e8afd24dbe761cc Author: Xikui Wang Date: Tue Apr 25 13:31:16 2017 -0700 Fix NULL/MISSING field value handing in Java UDF Add NULL/MISSING value support in Java UDF. This fixed the NPE when there is a NULL/MISSING field in a record. Reviewed-on: https://asterix-gerrit.ics.uci.edu/1695 Sonar-Qube: Jenkins Tested-by: Jenkins BAD: Jenkins Integration-Tests: Jenkins Reviewed-by: Yingyi Bu commit 37dba5e4d45ce3c86e7a797e0b1bcb101231e4b2 Author: Till Westmann Date: Mon Apr 24 13:43:38 2017 -0700 Remove IOMVisitor and its only (unused) implementation Reviewed-on: https://asterix-gerrit.ics.uci.edu/1699 Sonar-Qube: Jenkins Tested-by: Jenkins Integration-Tests: Jenkins Reviewed-by: Yingyi Bu commit 5449e85ab32d51d88e29b1f50d8450d1bd7a05bd Author: Taewoo Kim Date: Sun Apr 23 17:29:52 2017 -0700 Index-Only Plan Patch Set 4: Index SearchCursor adjustment - Modify the search cursors to maintain the result of proceed() since the result will be required to check whether the given tuple is qualified to be part of an index-only plan result or not. More details will be followed in the next patch set. - Fix the search cursors to call cancel() correctly. Reviewed-on: https://asterix-gerrit.ics.uci.edu/1630 Tested-by: Jenkins BAD: Jenkins In
Change in asterixdb[master]: Implemented LSM disk components alignment for datasets using...
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1725 to look at the new patch set (#6). Change subject: Implemented LSM disk components alignment for datasets using CorrelatedPrefixMergePolicy. The basic idea is to always flush/merge all indexes of a correlated dataset together into a group. Design doc:https://cwiki.apache.org/confluence/display/ASTERIXDB/L .. Implemented LSM disk components alignment for datasets using CorrelatedPrefixMergePolicy. The basic idea is to always flush/merge all indexes of a correlated dataset together into a group. Design doc:https://cwiki.apache.org/confluence/display/ASTERIXDB/LSM+Component+Alignment+Design+Doc Change-Id: I64bf34e255def72adc73b9f87cfa628a172ea694 --- A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/results/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.adm M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationGroupCallback.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMergeOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperationGroup.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperationGroupCallback.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperationScheduler.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMMergePolicy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-ls
Change in asterixdb[master]: Implemented LSM disk components alignment for datasets using...
Luo Chen has posted comments on this change. Change subject: Implemented LSM disk components alignment for datasets using CorrelatedPrefixMergePolicy. The basic idea is to always flush/merge all indexes of a correlated dataset together into a group. Design doc:https://cwiki.apache.org/confluence/display/ASTERIXDB/L .. Patch Set 6: > Chen Luo, > Can you look at the style of commit message in merged changes and > update accordingly? Sorry for that. Just cleaned up the commit message. -- To view, visit https://asterix-gerrit.ics.uci.edu/1725 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I64bf34e255def72adc73b9f87cfa628a172ea694 Gerrit-PatchSet: 6 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: No
Change in asterixdb[master]: Implemented LSM disk components alignment
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1725 to look at the new patch set (#7). Change subject: Implemented LSM disk components alignment .. Implemented LSM disk components alignment Implemented LSM disk components alignment for datasets using CorrelatedPrefixMergePolicy. The basic idea is to always flush/merge all indexes of a correlated dataset together into a group. Design doc:https://cwiki.apache.org/confluence/display/ ASTERIXDB/LSM+Component+Alignment+Design+Doc Change-Id: I64bf34e255def72adc73b9f87cfa628a172ea694 --- A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/results/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.adm M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationGroupCallback.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMergeOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperationGroup.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperationGroupCallback.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperationScheduler.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMMergePolicy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AsynchronousScheduler.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/a
Change in asterixdb[master]: Implemented LSM disk components alignment
Luo Chen has posted comments on this change. Change subject: Implemented LSM disk components alignment .. Patch Set 7: > Looks much better now. Still need small changes. > > Look at: https://cwiki.apache.org/confluence/display/ASTERIXDB/Formatting > > Mainly. > 1- Subject line (~50 chars) > 2. Empty line after. > 3. Each line after that is less than 77 chars. > > Will look at the actual files after the weekly :) Should be fixed now :) -- To view, visit https://asterix-gerrit.ics.uci.edu/1725 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I64bf34e255def72adc73b9f87cfa628a172ea694 Gerrit-PatchSet: 7 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: No
Change in asterixdb[master]: Implemented LSM disk components alignment
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1725 to look at the new patch set (#8). Change subject: Implemented LSM disk components alignment .. Implemented LSM disk components alignment Implemented LSM disk components alignment for datasets using CorrelatedPrefixMergePolicy. The basic idea is to always flush/merge all indexes of a correlated dataset together into a group. Design doc:https://cwiki.apache.org/confluence/display/ ASTERIXDB/LSM+Component+Alignment+Design+Doc Change-Id: I64bf34e255def72adc73b9f87cfa628a172ea694 --- A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/results/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.adm M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationGroupCallback.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btr
Change in asterixdb[master]: Implemented LSM disk components alignment
Luo Chen has posted comments on this change.
Change subject: Implemented LSM disk components alignment
..
Patch Set 8:
(6 comments)
> (6 comments)
https://asterix-gerrit.ics.uci.edu/#/c/1725/7/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
File
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java:
PS7, Line 332:
> Could this name be changed? It is a bit misleading, it implies to me at lea
What about 'isCorrelated'?
PS7, Line 337:
> Is there a better way to get this info? Shouldn't it be visible from somewh
I carried 'partition' info when constructing LSMIndex (which causes a lot of
code changes). Would these changes be OK?
https://asterix-gerrit.ics.uci.edu/#/c/1725/7/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AsynchronousScheduler.java
File
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AsynchronousScheduler.java:
PS7, Line 59: waitingOperationGroups
> Isnt there concurrent access to this variable?
There should be no concurrent access to this variable (all accesses of this map
is synchronized under the executor object).
PS7, Line 97: offer
> Why offer?
Just to be consistent with the previous 'runningFlushOperations' (line 80).
Since it is LinkedList, the offer operation would always succeed.
https://asterix-gerrit.ics.uci.edu/#/c/1725/7/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
File
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java:
PS7, Line 458: );
> Why 0?
If it is a flush operation, then ctx.getComponentHolder() would only contain
the exact one memory component to be flushed. (This function is summarized from
'getAndEnterComponents' for checking whether a component is ready to flush).
https://asterix-gerrit.ics.uci.edu/#/c/1725/7/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIOOperationGroup.java
File
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIOOperationGroup.java:
PS7, Line 92: }
: }
: }
: if (!ready) {
: return null;
: }
: List indexes = new
ArrayList<>(indexSet.size());
: List operations = new
ArrayList<>(indexSet.size());
:
: for (ILSMIndex lsmIndex : indexSet) {
: if (lsmIndex.getPartition() == partitionID) {
: //get resource
: //schedule flush after update
: ILSMIOOperation operation =
:
indexAccessors.get(indexes.size()).createFlushOperation(lsmIndex.getIOOperationCallback());
: if (operation == null) {
: throw new IllegalStateException("Inconsistent
memory component state for dataset " + datasetID);
: }
: operations.add(operation);
: indexes.add(lsmIndex);
: }
: }
:
: return new LSMIOOperationGroup(datasetID, partitionID,
LSMIOOpertionType.FLUSH, indexes, indexAccessors,
: operations, callback);
: }
> Whys all this down here? Can we move it to the top of the class?
Done
--
To view, visit https://asterix-gerrit.ics.uci.edu/1725
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I64bf34e255def72adc73b9f87cfa628a172ea694
Gerrit-PatchSet: 8
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Luo Chen
Gerrit-Reviewer: Ian Maxon
Gerrit-Reviewer: Jenkins
Gerrit-Reviewer: Jianfeng Jia
Gerrit-Reviewer: Luo Chen
Gerrit-Reviewer: Till Westmann
Gerrit-Reviewer: Yingyi Bu
Gerrit-Reviewer: abdullah alamoudi
Gerrit-HasComments: Yes
Change in asterixdb[master]: Implemented LSM disk components alignment
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1725 to look at the new patch set (#9). Change subject: Implemented LSM disk components alignment .. Implemented LSM disk components alignment Implemented LSM disk components alignment for datasets using CorrelatedPrefixMergePolicy. The basic idea is to always flush/merge all indexes of a correlated dataset together into a group. Design doc:https://cwiki.apache.org/confluence/display/ ASTERIXDB/LSM+Component+Alignment+Design+Doc Change-Id: I64bf34e255def72adc73b9f87cfa628a172ea694 --- A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/results/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.adm M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationGroupCallback.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btr
Change in asterixdb[master]: Implemented LSM disk components alignment
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1725 to look at the new patch set (#10). Change subject: Implemented LSM disk components alignment .. Implemented LSM disk components alignment Implemented LSM disk components alignment for datasets using CorrelatedPrefixMergePolicy. The basic idea is to always flush/merge all indexes of a correlated dataset together into a group. Design doc:https://cwiki.apache.org/confluence/display/ ASTERIXDB/LSM+Component+Alignment+Design+Doc Change-Id: I64bf34e255def72adc73b9f87cfa628a172ea694 --- A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/results/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.adm M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationGroupCallback.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/bt
Change in asterixdb[master]: Implemented LSM disk components alignment
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1725 to look at the new patch set (#11). Change subject: Implemented LSM disk components alignment .. Implemented LSM disk components alignment Implemented LSM disk components alignment for datasets using CorrelatedPrefixMergePolicy. The basic idea is to always flush/merge all indexes of a correlated dataset together into a group. Design doc:https://cwiki.apache.org/confluence/display/ ASTERIXDB/LSM+Component+Alignment+Design+Doc Change-Id: I64bf34e255def72adc73b9f87cfa628a172ea694 --- A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/results/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.adm M asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationGroupCallback.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/im
Change in asterixdb[master]: Implemented LSM disk components alignment
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1725 to look at the new patch set (#12). Change subject: Implemented LSM disk components alignment .. Implemented LSM disk components alignment Implemented LSM disk components alignment for datasets using CorrelatedPrefixMergePolicy. The basic idea is to always flush/merge all indexes of a correlated dataset together into a group. Design doc:https://cwiki.apache.org/confluence/display/ ASTERIXDB/LSM+Component+Alignment+Design+Doc Change-Id: I64bf34e255def72adc73b9f87cfa628a172ea694 --- A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/results/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.adm M asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationGroupCallback.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/im
Change in asterixdb[master]: Implemented LSM disk components alignment
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1725 to look at the new patch set (#13). Change subject: Implemented LSM disk components alignment .. Implemented LSM disk components alignment Implemented LSM disk components alignment for datasets using CorrelatedPrefixMergePolicy. The basic idea is to always flush/merge all indexes of a correlated dataset together into a group. Design doc:https://cwiki.apache.org/confluence/display/ ASTERIXDB/LSM+Component+Alignment+Design+Doc Change-Id: I64bf34e255def72adc73b9f87cfa628a172ea694 --- A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/results/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.adm M asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationGroupCallback.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/im
Change in asterixdb[master]: Implemented LSM disk components alignment
Luo Chen has posted comments on this change. Change subject: Implemented LSM disk components alignment .. Patch Set 13: (2 comments) > (5 comments) https://asterix-gerrit.ics.uci.edu/#/c/1725/7/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java File hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java: PS7, Line 337: > I'm not sure that's quite right either. Why does the index need to know abo When talking about components correlation, what it really means is that all disk components of all indexes of a dataset partition should be correlated. Indexes of different partitions of a dataset (even if they are in the same NC) could have different number of components. Suppose a NC has two partitions P1 and P2 for a dataset. They share the same memory budget for memory components and the same PrimaryOperationTracker. Whenever P1 or P2 is full would make all indexes of P1 and P2 to be flushed. However, when memory components of P1 are full, it is possible that memory components of P2 are not flashable, e.g., they are empty or flushing, and thus there's nothing to be flushed for P2. Thus, we need to group the flush operation of all indexes of one partition, instead of all indexes of all partitions blindly. https://asterix-gerrit.ics.uci.edu/#/c/1725/7/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java File hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java: PS7, Line 458: ex > Ok, well, maybe a comment or something to make that more explicit would be Done -- To view, visit https://asterix-gerrit.ics.uci.edu/1725 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I64bf34e255def72adc73b9f87cfa628a172ea694 Gerrit-PatchSet: 13 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: Yes
Change in asterixdb[master]: Implemented LSM disk components alignment
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1725 to look at the new patch set (#15). Change subject: Implemented LSM disk components alignment .. Implemented LSM disk components alignment Implemented LSM disk components alignment for datasets using CorrelatedPrefixMergePolicy. The basic idea is to always flush/merge all indexes of a correlated dataset together into a group. Design doc:https://cwiki.apache.org/confluence/display/ ASTERIXDB/LSM+Component+Alignment+Design+Doc Change-Id: I64bf34e255def72adc73b9f87cfa628a172ea694 --- A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/results/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.adm M asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationGroupCallback.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/im
Change in asterixdb[master]: Implemented LSM disk components alignment
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1725 to look at the new patch set (#17). Change subject: Implemented LSM disk components alignment .. Implemented LSM disk components alignment Implemented LSM disk components alignment for datasets using CorrelatedPrefixMergePolicy. The basic idea is to always flush/merge all indexes of a correlated dataset together into a group. Design doc:https://cwiki.apache.org/confluence/display/ ASTERIXDB/LSM+Component+Alignment+Design+Doc Change-Id: I64bf34e255def72adc73b9f87cfa628a172ea694 --- A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/results/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.adm M asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationGroupCallback.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/im
Change in asterixdb[master]: Implemented LSM disk components alignment
Luo Chen has posted comments on this change. Change subject: Implemented LSM disk components alignment .. Patch Set 17: (20 comments) > (29 comments) > > Added some "non-technical" comments. https://asterix-gerrit.ics.uci.edu/#/c/1725/16/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.aql File asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.aql: PS16, Line 21: > White space Done PS16, Line 22: > WS Done https://asterix-gerrit.ics.uci.edu/#/c/1725/16/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.aql File asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.aql: PS16, Line 21: policy > White space. Sorry that not quite familiar with the format requirements. Now they're fixed. PS16, Line 21: > Ws Done https://asterix-gerrit.ics.uci.edu/#/c/1725/16/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.aql File asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.aql: PS16, Line 21: > WS Done PS16, Line 22: > WS Done PS16, Line 26: > WS Done https://asterix-gerrit.ics.uci.edu/#/c/1725/16/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.aql File asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.aql: PS16, Line 21: > WS Done PS16, Line 22: > WS Done PS16, Line 27: > WS Done https://asterix-gerrit.ics.uci.edu/#/c/1725/16/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.sqlpp File asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.sqlpp: PS16, Line 21: > WS Done PS16, Line 22: > WS Done https://asterix-gerrit.ics.uci.edu/#/c/1725/16/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.sqlpp File asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.sqlpp: PS16, Line 21: > WS Done PS16, Line 22: > WS Done PS16, Line 25: > WS Done https://asterix-gerrit.ics.uci.edu/#/c/1725/16/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.sqlpp File asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.sqlpp: PS16, Line 21: > WS Done PS16, Line 22: > WS Done PS16, Line 27: > WS Done https://asterix-gerrit.ics.uci.edu/#/c/1725/16/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java File asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java: PS16, Line 44: Logger > Why log4j instead java.util.logging? Logger is not actually used. Removed this import. PS16, Line 95: IllegalStateException > Use error code and align with the exception guideline: Got it. BTW, I saw "All message templates should be tested.", but where should we test them? (searched the codebase, but didn't find any tests for them...) -- To view, visit https://asterix-gerrit.ics.uci.edu/1725 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I64bf34e255def72adc73b9f87cfa628a172ea694 Gerrit-PatchSet: 17 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: Yes
Change in asterixdb[master]: Implemented LSM disk components alignment
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1725 to look at the new patch set (#18). Change subject: Implemented LSM disk components alignment .. Implemented LSM disk components alignment Implemented LSM disk components alignment for datasets using CorrelatedPrefixMergePolicy. The basic idea is to always flush/merge all indexes of a correlated dataset together into a group. Design doc:https://cwiki.apache.org/confluence/display/ ASTERIXDB/LSM+Component+Alignment+Design+Doc Change-Id: I64bf34e255def72adc73b9f87cfa628a172ea694 --- A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/results/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.adm M asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationGroupCallback.java M asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/st
Change in asterixdb[master]: Implemented LSM disk components alignment
Luo Chen has posted comments on this change.
Change subject: Implemented LSM disk components alignment
..
Patch Set 18:
(4 comments)
> (24 comments)
>
> half way through this but I think this is enough for this patch...
https://asterix-gerrit.ics.uci.edu/#/c/1725/17/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java
File
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java:
PS17, Line 94: lsmIndex.getPartition() == index.getPartition()) {
: if (numComponents !=
lsmIndex.getImmutableComponents().size()) {
:
RuntimeDataException.create(ErrorCode.INDEX_NOT_ALIGNED, lsmIndex.toString(),
index.toString());
: }
> This will end up being thrown a lot. For example, if there is an secondary
Got it! The current implementation based on a critical assumption that whenever
the merge policy is called, the primary index should always have the same
number of disk components w.r.t. secondaries. However, it seems this assumption
is very difficult to achieve, and makes the whole solution very brittle (once
the assumption is broken, the whole thing won't work...)
I might need to reconsider the whole solution based on this fact...
https://asterix-gerrit.ics.uci.edu/#/c/1725/17/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
File
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java:
PS17, Line 79: public synchronized ILSMIndex getPrimaryIndex() {
: Set indexSet = getDatasetIndexes();
: for (ILSMIndex index : indexSet) {
: if (index.isPrimaryIndex()) {
: return index;
: }
: }
: return null;
: }
> What is the intended behavior of this method? can we add a comment?
Simply get the primary index for this dataset (as a helper method).
https://asterix-gerrit.ics.uci.edu/#/c/1725/17/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
File
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java:
PS17, Line 411: Set indexSet = dsInfo.getDatasetIndexes();
: ILSMIndex primaryIndex = dsInfo.getPrimaryIndex();
: if (primaryIndex.isCorrelated()) {
: List operationGroups =
:
LSMIOOperationGroup.create(dsInfo.getDatasetID(), indexSet, new
LSMIOOperationGroupCallback());
: for (ILSMIOOperationGroup operationGroup :
operationGroups) {
:
primaryIndex.getIOScheduler().scheduleOperationGroup(operationGroup);
: }
: } else {
: for (ILSMIndex lsmIndex : indexSet) {
: ILSMIndexAccessor accessor =
:
lsmIndex.createAccessor(NoOpOperationCallback.INSTANCE,
NoOpOperationCallback.INSTANCE);
:
accessor.scheduleFlush(lsmIndex.getIOOperationCallback());
: }
> This kind of code with if else is bad. essentially, the code should be the
Ideally, it should be. But the current solution requires all indexes be flushed
together in a group, which thus requires such if-else statements.
https://asterix-gerrit.ics.uci.edu/#/c/1725/17/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
File
asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java:
PS17, Line 64: partition()
> Storage in Hyracks has been designed to be be agnostic to storage partition
Got it. The current solution requires knowing the partition id of each index.
Maybe I can use the recent change (separate index creation with access) to get
rid of this.
--
To view, visit https://asterix-gerrit.ics.uci.edu/1725
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I64bf34e255def72adc73b9f87cfa628a172ea694
Gerrit-PatchSet: 18
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Luo Chen
Gerrit-Reviewer: Ian Maxon
Gerrit-Reviewer: Jenkins
Gerrit-Reviewer: Jianfeng Jia
Gerrit-Reviewer: Luo Chen
Gerrit-Reviewer: Till Westmann
Gerrit-Reviewer: Yingyi Bu
Gerrit-Reviewer: abdullah alamoudi
Gerrit-HasComments: Yes
Change in asterixdb[master]: Implemented LSM disk components alignment
Luo Chen has posted comments on this change. Change subject: Implemented LSM disk components alignment .. Patch Set 18: > I think we need to consider a situation where a secondary index is > built at a later time. For example, I can create a dataset with a > secondary index A, after one day, I can create an another secondary > index B. Yes. That part is the second step of this issue. Currently, when we create a new secondary index, we would scan everything from the primary, and then do a merge sort, and then create one giant disk component for the new secondary using the bulk load operator. Now we need to create a new disk component for each disk component of the primary. -- To view, visit https://asterix-gerrit.ics.uci.edu/1725 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I64bf34e255def72adc73b9f87cfa628a172ea694 Gerrit-PatchSet: 18 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Taewoo Kim Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: No
Change in asterixdb[master]: Implemented LSM disk components alignment
Luo Chen has posted comments on this change. Change subject: Implemented LSM disk components alignment .. Patch Set 18: > >> Now we need to create a new disk component > >> for each disk component of the primary. > > To get this work, when you are at a tuple in a disk component, you > need to search it in all previous disk components to see whether > the secondary key value changes? If secondary key value changes, > you need to delete the old entry in the 2ndary index (by adding an > anti-matter entry)? > > > An alternative might be to flush and compact primary index to one > disk component before bulk loading a 2ndary index. I think the first comment is to address the issue caused by upsert? I haven't considered too much about implementing newly created secondary index, and this would be a separate patch after the correlated merge policy. By the way, it might not be desirable to merge all disk components into one, because for some applications like Cloudberry, it's desirable to have many small disk components and use filters to filter out most of them during the query. I would need to check with the user (i.e., Cloudberry) to see whether this is a acceptable side-effect. -- To view, visit https://asterix-gerrit.ics.uci.edu/1725 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I64bf34e255def72adc73b9f87cfa628a172ea694 Gerrit-PatchSet: 18 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Taewoo Kim Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: No
Change in asterixdb[master]: Implemented Disk Components Alignment Based on IDs
Luo Chen has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/1761
Change subject: Implemented Disk Components Alignment Based on IDs
..
Implemented Disk Components Alignment Based on IDs
- Added IDs (using LSN) for disk components. When a disk component is
flushed, its initial ID is set as the LSN. When components are merged,
the result ID is the union of all IDs to be merged.
- Changed the correlated merge policy to correlate disk components of
the primary and secondaries using IDs.
Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e
---
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetResource.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/IndexInfo.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallback.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java
A
asterixdb/asterix-common/src/test/java/org/apache/asterix/test/context/CorrelatedPrefixMergePolicyTest.java
M
asterixdb/asterix-common/src/test/java/org/apache/asterix/test/memory/ConcurrentFramePoolUnitTest.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java
A
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentID.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java
A
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMDiskComponentID.java
16 files changed, 738 insertions(+), 81 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/61/1761/1
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java
index cd1d95e..d4a80ea 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java
@@ -31,9 +31,11 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import
org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.ComponentState;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentID;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
+import org.apache.hyracks.storage.am.lsm.common.impls.PrefixMergePolicy;
public class CorrelatedPrefixMergePolicy implements ILSMMergePolicy {
@@ -57,66 +59,30 @@
// all such components for which the sum of their sizes exceeds
MaxMrgCompSz. Schedule a merge of those components into a new component.
// 2. If a merge from 1 doesn't happen, see if the set of candidate
components for merging exceeds MaxTolCompCnt. If so, schedule
// a merge all of the current candidates into a new single component.
-List immutableComponents = new
ArrayList<>(index.getImmutableComponents());
-// Reverse the components order so that we look at components from
oldest to newest.
-Collections.reverse(immutableComponents);
-for (ILSMDiskComponent c : immutableComponents) {
-if (c.getState() != ComponentState.READABLE_UNWRITABLE) {
+if (fullMergeIsRequested) {
+//full merge request is handled by each index separately, since it
is possible that
+//when a primary index wants to send full merge requests for all
secondaries,
+//one secondary index is being merged and the request cannot be
scheduled
+ArrayList immutableComponents = new
ArrayList<>(index.getImmutableComponents());
+
Change in asterixdb[master]: Implemented Disk Components Alignment Based on IDs
Luo Chen has uploaded a new patch set (#2). Change subject: Implemented Disk Components Alignment Based on IDs .. Implemented Disk Components Alignment Based on IDs - Added IDs (using LSN) for disk components. When a disk component is flushed, its initial ID is set as the LSN. When components are merged, the result ID is the union of all IDs to be merged. - Changed the correlated merge policy to correlate disk components of the primary and secondaries using IDs. Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e --- M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetResource.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/IndexInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallback.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java A asterixdb/asterix-common/src/test/java/org/apache/asterix/test/context/CorrelatedPrefixMergePolicyTest.java M asterixdb/asterix-common/src/test/java/org/apache/asterix/test/memory/ConcurrentFramePoolUnitTest.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentID.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMDiskComponentID.java 16 files changed, 719 insertions(+), 62 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/61/1761/2 -- To view, visit https://asterix-gerrit.ics.uci.edu/1761 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newpatchset Gerrit-Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Jenkins
Change in asterixdb[master]: Implemented Disk Components Alignment Based on IDs
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1761 to look at the new patch set (#3). Change subject: Implemented Disk Components Alignment Based on IDs .. Implemented Disk Components Alignment Based on IDs - Added IDs (using LSN) for disk components. When a disk component is flushed, its initial ID is set as the LSN. When components are merged, the result ID is the union of all IDs to be merged. - Changed the correlated merge policy to correlate disk components of the primary and secondaries using IDs. Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e --- M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetResource.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/IndexInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallback.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java A asterixdb/asterix-common/src/test/java/org/apache/asterix/test/context/CorrelatedPrefixMergePolicyTest.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentID.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMDiskComponentID.java 15 files changed, 718 insertions(+), 61 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/61/1761/3 -- To view, visit https://asterix-gerrit.ics.uci.edu/1761 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newpatchset Gerrit-Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e Gerrit-PatchSet: 3 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Jenkins
Change in asterixdb[master]: Change logical plan to apply filter from 2ndary index
Luo Chen has posted comments on this change. Change subject: Change logical plan to apply filter from 2ndary index .. Patch Set 7: (4 comments) https://asterix-gerrit.ics.uci.edu/#/c/1727/7/.gitignore File .gitignore: Why change this file? https://asterix-gerrit.ics.uci.edu/#/c/1727/7/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java File asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java: PS7, Line 320: assignFilterFromSecondaryUnnest This function is not being used? Could it be deleted? https://asterix-gerrit.ics.uci.edu/#/c/1727/7/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/delete/delete.1.ddl.aql File asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/delete/delete.1.ddl.aql: Line 1: /* Where did you include this test? I thought we should change the some testsuite files like "testsuite.xml" to include them? https://asterix-gerrit.ics.uci.edu/#/c/1727/7/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/upsert/upsert.1.ddl.aql File asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/upsert/upsert.1.ddl.aql: Line 1: /* It's better to add the SQLPP version of the tests? -- To view, visit https://asterix-gerrit.ics.uci.edu/1727 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I0e2fe0208662e5dcd49d1a22bfb58f96533e9497 Gerrit-PatchSet: 7 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Jianfeng Jia Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: Yes
Change in asterixdb[master]: Implemented Disk Components Alignment Based on IDs
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1761 to look at the new patch set (#4). Change subject: Implemented Disk Components Alignment Based on IDs .. Implemented Disk Components Alignment Based on IDs - Added IDs (using LSN) for disk components. When a disk component is flushed, its initial ID is set as the LSN. When components are merged, the result ID is the union of all IDs to be merged. - Changed the correlated merge policy to correlate disk components of the primary and secondaries using IDs. Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e --- M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetResource.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/IndexInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallback.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java A asterixdb/asterix-common/src/test/java/org/apache/asterix/test/context/CorrelatedPrefixMergePolicyTest.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentID.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMDiskComponentID.java 15 files changed, 717 insertions(+), 61 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/61/1761/4 -- To view, visit https://asterix-gerrit.ics.uci.edu/1761 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newpatchset Gerrit-Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e Gerrit-PatchSet: 4 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Jenkins
Change in asterixdb[master]: Implemented LSM disk components alignment
Luo Chen has abandoned this change. Change subject: Implemented LSM disk components alignment .. Abandoned -- To view, visit https://asterix-gerrit.ics.uci.edu/1725 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: abandon Gerrit-Change-Id: I64bf34e255def72adc73b9f87cfa628a172ea694 Gerrit-PatchSet: 18 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Taewoo Kim Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi
Change in asterixdb[master]: Implemented Disk Components Alignment Based on IDs
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1761 to look at the new patch set (#5). Change subject: Implemented Disk Components Alignment Based on IDs .. Implemented Disk Components Alignment Based on IDs - Added IDs (using LSN) for disk components. When a disk component is flushed, its initial ID is set as the LSN. When components are merged, the result ID is the union of all IDs to be merged. - Changed the correlated merge policy to correlate disk components of the primary and secondaries using IDs. Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e --- M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetResource.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/IndexInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallback.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java A asterixdb/asterix-common/src/test/java/org/apache/asterix/test/context/CorrelatedPrefixMergePolicyTest.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentID.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMDiskComponentID.java 15 files changed, 770 insertions(+), 61 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/61/1761/5 -- To view, visit https://asterix-gerrit.ics.uci.edu/1761 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newpatchset Gerrit-Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e Gerrit-PatchSet: 5 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi
Change in asterixdb[master]: Implemented Disk Components Alignment Based on IDs
Luo Chen has posted comments on this change. Change subject: Implemented Disk Components Alignment Based on IDs .. Patch Set 5: (4 comments) > (4 comments) > > A few questions https://asterix-gerrit.ics.uci.edu/#/c/1761/4/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java File asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java: PS4, Line 105: * case 1. : * if merga > Can there be a toplevel comment describing what case 1/2/3 are? Done PS4, Line 180: totalSize += componentSize; > Why call this with a set of indices, and the partition, and filter out the Done https://asterix-gerrit.ics.uci.edu/#/c/1761/4/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java File asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java: PS4, Line 114: g id = getComponentLSN(null); > Wait, what? Why getComonentLSN of a null value? If oldComponents == null, then getComponentLSN would treat it as a flush operation, and return the LSN for the flushed component. I added some comments for this. https://asterix-gerrit.ics.uci.edu/#/c/1761/4/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java File hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java: PS4, Line 108: //TODO: do we need to throw an exception when ID is not found? > If you can make it so that it fails safely that would be good. This way the Currently, if a disk component is formed by flushing, then its componentID should be set. However, one exception is that for bulk-loaded components, such as those created by loading a dataset, the componentID is currently missing. If this happens, AbstractLSMIOOperationCallback would log it. -- To view, visit https://asterix-gerrit.ics.uci.edu/1761 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e Gerrit-PatchSet: 5 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: Yes
Change in asterixdb[master]: Implemented Disk Components Alignment Based on IDs
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1761 to look at the new patch set (#6). Change subject: Implemented Disk Components Alignment Based on IDs .. Implemented Disk Components Alignment Based on IDs - Added IDs (using LSN) for disk components. When a disk component is flushed, its initial ID is set as the LSN. When components are merged, the result ID is the union of all IDs to be merged. - Changed the correlated merge policy to correlate disk components of the primary and secondaries using IDs. Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e --- M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetResource.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/IndexInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallback.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java A asterixdb/asterix-common/src/test/java/org/apache/asterix/test/context/CorrelatedPrefixMergePolicyTest.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentID.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMDiskComponentID.java 15 files changed, 748 insertions(+), 61 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/61/1761/6 -- To view, visit https://asterix-gerrit.ics.uci.edu/1761 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newpatchset Gerrit-Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e Gerrit-PatchSet: 6 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi
Change in asterixdb[master]: ASTERIXDB-1917: FLUSH_LSN for disk components is not correct...
Luo Chen has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/1771
Change subject: ASTERIXDB-1917: FLUSH_LSN for disk components is not correctly
set
..
ASTERIXDB-1917: FLUSH_LSN for disk components is not correctly set
-Fixed a bug that FLUSH_LSN for flushed disk components is not
correctly set (not increasing) when an NC has multiple partitions.
-Added LSMIOOperationCallback unit tests to cover this bug
Change-Id: If438e34f8f612458d81f618eea04c0c72c49a9fe
---
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
A
asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeIOOperationCallbackTest.java
A
asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackTest.java
A
asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMInvertedIndexIOOperationCallbackTest.java
A
asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMRTreeIOOperationCallbackTest.java
5 files changed, 344 insertions(+), 2 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/71/1771/1
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
index f903b65..d04443a 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
@@ -107,8 +107,14 @@
return pointable.getLength() == 0 ? INVALID : pointable.longValue();
}
-public void updateLastLSN(long lastLSN) {
-mutableLastLSNs[writeIndex] = lastLSN;
+public synchronized void updateLastLSN(long lastLSN) {
+if (!flushRequested[writeIndex]) {
+//if the memory component pointed by writeIndex is being flushed,
we should ignore this update call
+//since otherwise the original LSN is overwritten.
+//Moreover, since the memory component is already being flushed,
the next scheduleFlush request must fail.
+//See https://issues.apache.org/jira/browse/ASTERIXDB-1917
+mutableLastLSNs[writeIndex] = lastLSN;
+}
}
public void setFirstLSN(long firstLSN) {
diff --git
a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeIOOperationCallbackTest.java
b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeIOOperationCallbackTest.java
new file mode 100644
index 000..bc206e2
--- /dev/null
+++
b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeIOOperationCallbackTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.test.ioopcallbacks;
+
+import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.junit.Assert;
+import org.mockito.Mockito;
+
+import junit.framework.TestCase;
+
+public class LSMBTreeIOOperationCallbackTest extends TestCase {
+
+public void testNormalSequence() {
+try {
+LSMBTreeIOOperationCallback callback = new
LSMBTreeIOOperationCallback();
+callback.setNumOfMutableComponents(2);
+
+//request to flush first component
+callback.updateLastLSN(1);
+callback.beforeOperation(LSMOperationType.FLUSH);
+
+//request to flush second component
+callback.updateLastLSN(2);
+callback.beforeOperation(LSMOperationType.FLUSH);
+
+Assert.assertEquals(1, callback.getComponentLSN(null));
+callback.afterFinalize(LSMOperationType.FLUSH,
Mockito.mock(ILSMDiskComponent.class));
+
+
Change in asterixdb[master]: Add LSM Disk Component Bulk Loader
Luo Chen has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/1773
Change subject: Add LSM Disk Component Bulk Loader
..
Add LSM Disk Component Bulk Loader
-Added ILSMDiskComponentBulkLoader interface, which is used to bulk
load an LSMDiskComponent with anti-matters
-Added ILSMDiskComponentWithBuddyBTreeBulkLoader, which is used to
bulk load an LSMDiskComponent with deleted-keys btrees
-Refactored LSM flush/merge/index bulk load operations to use
the LSMDiskComponentBulkLoader
Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb
---
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
A
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java
A
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java
A
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentWithBuddyBTreeBulkLoader.java
A
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java
A
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBTreeBulkLoader.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
A
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentBulkLoader.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
A
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentBulkLoader.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
A
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader.java
12 files changed, 873 insertions(+), 505 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/73/1773/1
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index 1c99d5a..d9e4da3 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -36,7 +36,6 @@
import
org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification;
import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor;
-import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeBulkLoader;
import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
import org.apache.hyracks.storage.am.common.api.IPageManager;
@@ -46,12 +45,12 @@
import org.apache.hyracks.storage.am.common.impls.AbstractSearchPredicate;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
-import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleReference;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
import
org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import
org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
@@ -426,12 +425,10 @@
RangePredicate nullPred = new RangePredicate(null, null, true, true,
null, null);
long numElements = 0L;
-BloomFilterSpecification bloomFilterSpec = null;
if (hasBloomFilter) {
//co
Change in asterixdb[master]: Add LSM Disk Component Bulk Loader
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1773 to look at the new patch set (#2). Change subject: Add LSM Disk Component Bulk Loader .. Add LSM Disk Component Bulk Loader -Added ILSMDiskComponentBulkLoader interface, which is used to bulk load an LSMDiskComponent with anti-matters -Added ILSMDiskComponentWithBuddyBTreeBulkLoader, which is used to bulk load an LSMDiskComponent with deleted-keys btrees -Refactored LSM flush/merge/index bulk load operations to use the LSMDiskComponentBulkLoader Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb --- M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentWithBuddyBTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBTreeBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader.java 12 files changed, 883 insertions(+), 507 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/73/1773/2 -- To view, visit https://asterix-gerrit.ics.uci.edu/1773 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newpatchset Gerrit-Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Jenkins
Change in asterixdb[master]: ASTERIXDB-1917: FLUSH_LSN for disk components is not correct...
Luo Chen has posted comments on this change. Change subject: ASTERIXDB-1917: FLUSH_LSN for disk components is not correctly set .. Patch Set 1: > The patch seems nice to me. I'm not sure I entirely understand the > proposed alternative. If I understand the bug correctly, the issue > here basically is that triggerFlushRequest will eagerly set the > LSN, which is incorrect because it can and will fail to flush when > a request is already pending. Regardless of whether we persist the > LSN in the component (which does make more sense to do), or inside > the callback, doesn't the same issue apply? Yes, I think even if we persist the LSN into the memory component, the same issue still applies since it is still possible to override the component LSN (just as this bug). Moreover, I think it's better to keep the IOOperationCallback as it is, otherwise the flush/merge API would carry some extra information (like LSN in this case) to make things work. -- To view, visit https://asterix-gerrit.ics.uci.edu/1771 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: If438e34f8f612458d81f618eea04c0c72c49a9fe Gerrit-PatchSet: 1 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Murtadha Hubail Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1917: FLUSH_LSN for disk components is not correct...
Luo Chen has posted comments on this change. Change subject: ASTERIXDB-1917: FLUSH_LSN for disk components is not correctly set .. Patch Set 1: > > > The patch seems nice to me. I'm not sure I entirely understand > > the > > > proposed alternative. If I understand the bug correctly, the > > issue > > > here basically is that triggerFlushRequest will eagerly set the > > > LSN, which is incorrect because it can and will fail to flush > > when > > > a request is already pending. Regardless of whether we persist > > the > > > LSN in the component (which does make more sense to do), or > > inside > > > the callback, doesn't the same issue apply? > > > > Yes, I think even if we persist the LSN into the memory > component, > > the same issue still applies since it is still possible to > override > > the component LSN (just as this bug). > > > > Moreover, I think it's better to keep the IOOperationCallback as > it > > is, otherwise the flush/merge API would carry some extra > > information (like LSN in this case) to make things work. > > I am not saying that the fix is incorrect. Here is what is > happening. > We have a memory component's metadata with the memory component. > This metadata already has pairs of key value. when we flush, we > move those key values to the disk component: > See: LSMBTree.java line 359: > flushingComponent.getMetadata().copy(component.getMetadata()); > > what about the LSN? it is not yet written to the disk component. > after the flush call returns, we do: > > operation.getCallback().afterOperation(LSMOperationType.FLUSH, > null, newComponent); > > If we put the lsn when we schedule the operation inside the > scheduleFlush() call, then it will be written to the metadata of > the disk component with the other key value pairs. if the lsn was > set later in the callback, it wouldn't affect the value that goes > to disk. This solution would work. But the problem is that then we have to put LSN information inside the LSMIndex (Hyracks codebase), where the LSN actually belongs to AsterixDB. I think the reason for this IOOperationCallback is to separate LSN from Hyracks. -- To view, visit https://asterix-gerrit.ics.uci.edu/1771 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: If438e34f8f612458d81f618eea04c0c72c49a9fe Gerrit-PatchSet: 1 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Murtadha Hubail Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: No
Change in asterixdb[master]: Implemented Disk Components Alignment Based on IDs
Luo Chen has posted comments on this change. Change subject: Implemented Disk Components Alignment Based on IDs .. Patch Set 6: (1 comment) https://asterix-gerrit.ics.uci.edu/#/c/1761/6/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java File asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java: PS6, Line 103: isMergeLagging( > I think that there is no need for this method after the introduction of com >From my experiments, we would still need this method for flow control. Here is >the reason. Suppose we have a dataset D with a BTree primary index P and InvertedIndex secondary index S. Note that the merge operation of S is much slower than P (one problem for InvertedIndex). Then a lot of data come in. The first time when the merge policy decides to merge P (say merge 5 components), it would also send a merge request on behalf of S. However, the second time when the merge policy decides to merge P, since the merge of S is still ongoing, S would be ignored for this time (there couldn't be two concurrent merge operations for the same index). The same thing could happen for the third time. After a while, when S finishes its merge operation, there could be a lot of disk components accumulated. The next time when the policy decides to merge, it could merge a lot of lagged disk components (from my observation it could be 50 or more). This problem would get even worse and worse, eventually blowing up the system. -- To view, visit https://asterix-gerrit.ics.uci.edu/1761 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e Gerrit-PatchSet: 6 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: Yes
Change in asterixdb[master]: Implemented Disk Components Alignment Based on IDs
Luo Chen has posted comments on this change. Change subject: Implemented Disk Components Alignment Based on IDs .. Patch Set 6: (1 comment) https://asterix-gerrit.ics.uci.edu/#/c/1761/6/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java File asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java: PS6, Line 103: isMergeLagging( > I see. the problem is that this delays flushes. while for the problem you'r To avoid this isMergeLagging method, another way we could do is that when the primary index needs to merge, it could wait for all secondary indexes to finish its ongoing merge operations so that the merge request would always succeed. However, this would slow down the performance since this essentially adds some extra synchronization to the system. BTW, even we could allow two or more concurrent merge operations of a single index, we would still need to do flow control, since if the merge operation is much slower than flush, then we would still get more and more disk components accumulated because the merge operation could finish on time. -- To view, visit https://asterix-gerrit.ics.uci.edu/1761 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e Gerrit-PatchSet: 6 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: Yes
Change in asterixdb[master]: Implemented Disk Components Alignment Based on IDs
Luo Chen has posted comments on this change. Change subject: Implemented Disk Components Alignment Based on IDs .. Patch Set 6: > (1 comment) Sorry a typo at the last sentence. It should be "because the merge operation couldn't finish on time." -- To view, visit https://asterix-gerrit.ics.uci.edu/1761 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e Gerrit-PatchSet: 6 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1917: FLUSH_LSN for disk components is not correct...
Luo Chen has posted comments on this change. Change subject: ASTERIXDB-1917: FLUSH_LSN for disk components is not correctly set .. Patch Set 2: Hi reviewers, it seems that I don't have to privilege to submit a patch (as I'm new to gerrit). Could one of the reviewers submit this patch? Thanks! -- To view, visit https://asterix-gerrit.ics.uci.edu/1771 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: If438e34f8f612458d81f618eea04c0c72c49a9fe Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Murtadha Hubail Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: No
Change in asterixdb[master]: Implemented Disk Components Alignment Based on IDs
Luo Chen has posted comments on this change. Change subject: Implemented Disk Components Alignment Based on IDs .. Patch Set 6: (1 comment) https://asterix-gerrit.ics.uci.edu/#/c/1761/6/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java File asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java: PS6, Line 103: isMergeLagging( > I am strongly opposed to delay of flushes for the sake of merges. Not sure Both solutions (with or without flow control) are solution OK to me. But without flow control, one consequence is that there could be more and more lagged disk components accumulated on the disk along with the data ingestion. But eventually when the ingestion stops, the disk components would be merged as expected. Any suggestions or comments from other reviewers? However, for the correlated merge policy, the merge operation shouldn't be triggered by any secondary index. The reason is that the secondary index disk component is usually much smaller than the primary index disk component. If the merge request is sent by the secondary index (for all indexes), then we would get much larger disk components for the primary, which breaks the criteria for the merge policy. For example, if we set the MaxMergableComponentSize=128M (this tells the merge policy that if any disk component is larger than this size, then never merge it again), when the primary index disk component reaches this limit, the corresponding secondary index disk component is only about 4-5M. -- To view, visit https://asterix-gerrit.ics.uci.edu/1761 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e Gerrit-PatchSet: 6 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: Yes
Change in asterixdb[master]: Implemented Disk Components Alignment Based on IDs
Luo Chen has posted comments on this change. Change subject: Implemented Disk Components Alignment Based on IDs .. Patch Set 7: (1 comment) https://asterix-gerrit.ics.uci.edu/#/c/1761/6/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java File asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java: PS6, Line 103: isMergeLagging( > That should be fine because the merge policy should always checks the prima I see. Then it makes sense. However, this still wouldn't fully solve the problem for the following reason. Let say after the 10 disk components are added to both indexes, the policy schedules a merge operation for the primary index, but misses the secondary index since it's still being merged. After the secondary index finishes it's merge operation, it would have more disk components than the merge policy tolerates. However, even suppose the secondary index could trigger the merge policy, since the merge policy only looks at the primary index, it still won't schedule the merge operation since the disk components of the primary index do not need to be merged now. Still, we would wait until the next flush operation for the primary index. This problem can be solved by let the merging policy (triggered at primary index) to wait for all secondary indexes to finish the ongoing merge operations. If that's the case, all indexes would be merged together, and no one would be left behind. But even for the current solution (without synchronization for merging but with flow control), this should be acceptable since the secondary index (more precisely, the inverted index) would at most miss one round of merge request. -- To view, visit https://asterix-gerrit.ics.uci.edu/1761 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e Gerrit-PatchSet: 7 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: Yes
Change in asterixdb[master]: Implemented Disk Components Alignment Based on IDs
Luo Chen has posted comments on this change. Change subject: Implemented Disk Components Alignment Based on IDs .. Patch Set 7: > (1 comment) I see. It looks clear to me now. I was mainly influenced by the previous implementation of the CorrelatedMergePolicy that the merge operations can only be triggered by the primary index. I'll fix this tomorrow. -- To view, visit https://asterix-gerrit.ics.uci.edu/1761 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e Gerrit-PatchSet: 7 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: No
Change in asterixdb[master]: Implemented Disk Components Alignment Based on IDs
Luo Chen has posted comments on this change. Change subject: Implemented Disk Components Alignment Based on IDs .. Patch Set 7: (1 comment) https://asterix-gerrit.ics.uci.edu/#/c/1761/6/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java File asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java: PS6, Line 103: isMergeLagging( > >> However, this still wouldn't fully solve the problem for the following r I was trying to implement this new solution today, which allows the merge operations be triggered by any index (but we only look at the primary index). However, after a few experiments, I found this solution is not doable because of deadlocks, and the detailed reason is as follows. Still, suppose we have a primary index P and secondary index S on a dataset partition. Suppose in thread T1, S wants to send merge requests for S and P. Now T1 is holding S.baseOpTracker, and waiting for P.primaryOpTracker. However, suppose in thread T2, P.primaryOpTracker wants to flush the dataset. Now T2 holds P.primaryOpTracker, and waits for S.baseOpTracker. Now we have a cycle here! Thus, the conclusion is that in the current context, it's better to let the primary coordinate all merge requests for all indexes (this won't cause the deadlock problem since the primary index must already hold primaryOpTracker). -- To view, visit https://asterix-gerrit.ics.uci.edu/1761 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e Gerrit-PatchSet: 7 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: Yes
Change in asterixdb[master]: Implemented Disk Components Alignment Based on IDs
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1761 to look at the new patch set (#8). Change subject: Implemented Disk Components Alignment Based on IDs .. Implemented Disk Components Alignment Based on IDs - Added IDs (using LSN) for disk components. When a disk component is flushed, its initial ID is set as the LSN. When components are merged, the result ID is the union of all IDs to be merged. - Changed the correlated merge policy to correlate disk components of the primary and secondaries using IDs. Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e --- M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetResource.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/IndexInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java A asterixdb/asterix-common/src/test/java/org/apache/asterix/test/context/CorrelatedPrefixMergePolicyTest.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentID.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMDiskComponentID.java 11 files changed, 744 insertions(+), 61 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/61/1761/8 -- To view, visit https://asterix-gerrit.ics.uci.edu/1761 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newpatchset Gerrit-Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e Gerrit-PatchSet: 8 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi
Change in asterixdb[master]: Add LSMDiskComponentBulkLoader
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1773 to look at the new patch set (#3). Change subject: Add LSMDiskComponentBulkLoader .. Add LSMDiskComponentBulkLoader -Added ILSMDiskComponentBulkLoader interface, which is used to bulk load an LSMDiskComponent with anti-matters -Added ILSMDiskComponentWithBuddyBTreeBulkLoader, which is used to bulk load an LSMDiskComponent with deleted-keys btrees -Refactored LSM flush/merge/index bulk load operations to use the LSMDiskComponentBulkLoader Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb --- M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentWithBuddyBTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBTreeBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader.java 14 files changed, 894 insertions(+), 516 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/73/1773/3 -- To view, visit https://asterix-gerrit.ics.uci.edu/1773 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newpatchset Gerrit-Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb Gerrit-PatchSet: 3 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Jenkins
Change in asterixdb[master]: Add LSMDiskComponentBulkLoader
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1773 to look at the new patch set (#4). Change subject: Add LSMDiskComponentBulkLoader .. Add LSMDiskComponentBulkLoader -Added ILSMDiskComponentBulkLoader interface, which is used to bulk load an LSMDiskComponent with anti-matters -Added ILSMDiskComponentWithBuddyBTreeBulkLoader, which is used to bulk load an LSMDiskComponent with deleted-keys btrees -Refactored LSM flush/merge/index bulk load operations to use the LSMDiskComponentBulkLoader Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb --- M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentWithBuddyBTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBTreeBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader.java 14 files changed, 904 insertions(+), 529 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/73/1773/4 -- To view, visit https://asterix-gerrit.ics.uci.edu/1773 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newpatchset Gerrit-Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb Gerrit-PatchSet: 4 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi
Change in asterixdb[master]: Add LSMDiskComponentBulkLoader
Luo Chen has posted comments on this change. Change subject: Add LSMDiskComponentBulkLoader .. Patch Set 4: > IIndexBulkLoader is exactly the same as ILSMDiskComponentBuilLoader. > why add a new interface and not use the existing one? There are similar, since currently the LSM index bulk load operation would also create one disk component. But here are a few differences: - In LSMDiskComponentBulkLoader, the component is created outside it, which gives the caller more freedom and makes it suitable for flush/merge/index bulk load operations. - The end() method of LSMDiskComponentBulkLoader does not put the component into the component list of the index immediately (while the index bulk loader does), and the caller has the freedom to handle the result component. - Its sub interface ILSMDiskComponentWithBuddyBTreeBulkLoader has a method called addDeletedTuple, which allows the caller to bulk load the buddy btree. - When we need to create a disk component (for flush/merge/index bulk load/create secondary index), the new interfaces allows us to only deal with two kinds of disk components (with anti-matter tuples/with buddy btree), instead of writing difference code for different lsm indexes. -- To view, visit https://asterix-gerrit.ics.uci.edu/1773 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb Gerrit-PatchSet: 4 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: No
Change in asterixdb[master]: Add LSMDiskComponentBulkLoader
Luo Chen has posted comments on this change. Change subject: Add LSMDiskComponentBulkLoader .. Patch Set 4: > I'm not sure I understand. The differences between IIndexBulkLoader > and ILSMDiskComponentBulkLoader seem to be all in the > implementation and not in the interface. So reusing/deriving from > the existing Hyracks interface in AsterixDB seems to be a good > idea. Though they have the same methods, the meaning for them is different. ILSMDiskComponentBulkLoader is used to bulk load a disk component, instead of a whole index (a disk component maybe shouldn't be called as an index?). If look at the code for flush and merge operations for LSMBTree/RTree/RTreeWithAntiMatterTuples/InvertedIndex, the basic structure is quite similar, and we are bulk loading a disk component (with anti-matter tuples or with deleted-btree). Without this LSMDiskComponentBulkLoader, then for each LSMIndex, we have to write different code to bulk load its corresponding disk component. ILSMDiskComponentBulkLoader (and its subinterface ILSMDiskComponentWithBuddyBTreeBulkLoader) provides a unified method to bulk load lsm disk components (since we currently only have two kinds of disk component) -- To view, visit https://asterix-gerrit.ics.uci.edu/1773 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb Gerrit-PatchSet: 4 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: No
Change in asterixdb[master]: Implemented Disk Components Alignment Based on IDs
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1761 to look at the new patch set (#9). Change subject: Implemented Disk Components Alignment Based on IDs .. Implemented Disk Components Alignment Based on IDs - Added IDs (using LSN) for disk components. When a disk component is flushed, its initial ID is set as the LSN. When components are merged, the result ID is the union of all IDs to be merged. - Changed the correlated merge policy to correlate disk components of the primary and secondaries using IDs. Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e --- M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetResource.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/IndexInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java A asterixdb/asterix-common/src/test/java/org/apache/asterix/test/context/CorrelatedPrefixMergePolicyTest.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentId.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMDiskComponentId.java 12 files changed, 753 insertions(+), 61 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/61/1761/9 -- To view, visit https://asterix-gerrit.ics.uci.edu/1761 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newpatchset Gerrit-Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e Gerrit-PatchSet: 9 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi
Change in asterixdb[master]: Implemented Disk Components Alignment Based on IDs
Luo Chen has posted comments on this change.
Change subject: Implemented Disk Components Alignment Based on IDs
..
Patch Set 8:
(11 comments)
> (13 comments)
https://asterix-gerrit.ics.uci.edu/#/c/1761/8/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java
File
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java:
PS8, Line 68: ArrayList
> ArrayList -> List
Done
PS8, Line 82: ArrayList
> ArrayList->List
Done
https://asterix-gerrit.ics.uci.edu/#/c/1761/8/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
File
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java:
PS8, Line 115: public
> public -> private ?
Done
PS8, Line 121: WARNING
> Should that be an exception? Is there any case that happen?
This could happen for bulk loaded index. At the end() method of LSM index bulk
loader (say LSMBTreeBulkLoader), afterOperation is called for that bulk loaded
component (and bulk loaded is treated as flush), but the FLUSH_LSN is not set
for bulk-loaded components.
If we can distinguish between FLUSH and BULK_LOAD operations, then we should
throw an exception.
PS8, Line 141: public
> public->private
Done
https://asterix-gerrit.ics.uci.edu/#/c/1761/8/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java
File
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java:
Line 49:
> Add java doc for the new interface method?
Done.
And I added java doc for ILSMDiskComponentID in that class (instead of this
method).
PS8, Line 50: getComponentID
> ID-> Id
Done
PS8, Line 50: ILSMDiskComponentID
> ID->Id
Done
https://asterix-gerrit.ics.uci.edu/#/c/1761/8/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java
File
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java:
PS8, Line 109: LSMDiskComponentID
> Make ILSMDiskComponentID a class member and construct it once in the constr
The problem is that for new disk components (flush or merge target), id is
missing when this component is being constructed. Thus, the id should be
non-final, and the caller needs to call setComponentId at a later time. Is this
acceptable?
https://asterix-gerrit.ics.uci.edu/#/c/1761/8/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMDiskComponentID.java
File
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMDiskComponentID.java:
PS8, Line 54: result
> return 31 * Long.hashCode(minId) + Long.hashCode(maxId) ?
Done
PS8, Line 64: if (obj == null) {
: return false;
: }
: if (getClass() != obj.getClass()) {
: return false;
: }
> replace 64 to 69 with
Done
--
To view, visit https://asterix-gerrit.ics.uci.edu/1761
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e
Gerrit-PatchSet: 8
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Luo Chen
Gerrit-Reviewer: Ian Maxon
Gerrit-Reviewer: Jenkins
Gerrit-Reviewer: Jianfeng Jia
Gerrit-Reviewer: Luo Chen
Gerrit-Reviewer: Till Westmann
Gerrit-Reviewer: Yingyi Bu
Gerrit-Reviewer: Yingyi Bu
Gerrit-Reviewer: abdullah alamoudi
Gerrit-HasComments: Yes
Change in asterixdb[master]: Implemented Disk Components Alignment Based on IDs
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1761 to look at the new patch set (#10). Change subject: Implemented Disk Components Alignment Based on IDs .. Implemented Disk Components Alignment Based on IDs - Added IDs (using LSN) for disk components. When a disk component is flushed, its initial ID is set as the LSN. When components are merged, the result ID is the union of all IDs to be merged. - Changed the correlated merge policy to correlate disk components of the primary and secondaries using IDs. Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e --- M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetResource.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/IndexInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java A asterixdb/asterix-common/src/test/java/org/apache/asterix/test/context/CorrelatedPrefixMergePolicyTest.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentId.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMDiskComponentId.java 11 files changed, 752 insertions(+), 61 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/61/1761/10 -- To view, visit https://asterix-gerrit.ics.uci.edu/1761 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newpatchset Gerrit-Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e Gerrit-PatchSet: 10 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi
Change in asterixdb[master]: Add LSMDiskComponentBulkLoader
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1773 to look at the new patch set (#5). Change subject: Add LSMDiskComponentBulkLoader .. Add LSMDiskComponentBulkLoader -Added ILSMDiskComponentBulkLoader interface, which is used to bulk load an LSMDiskComponent with anti-matters -Added ILSMDiskComponentWithBuddyBTreeBulkLoader, which is used to bulk load an LSMDiskComponent with deleted-keys btrees -Refactored LSM flush/merge/index bulk load operations to use the LSMDiskComponentBulkLoader Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb --- M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentWithBuddyBTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBTreeBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader.java 14 files changed, 909 insertions(+), 529 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/73/1773/5 -- To view, visit https://asterix-gerrit.ics.uci.edu/1773 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newpatchset Gerrit-Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb Gerrit-PatchSet: 5 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi
Change in asterixdb[master]: Add LSMDiskComponentBulkLoader
Luo Chen has posted comments on this change. Change subject: Add LSMDiskComponentBulkLoader .. Patch Set 5: > I'm not sure I follow. It seems that the methods do the exact same > thing. A consumer of the interface adds tuples to something and > then either marks a successful completion (end) or an error case > (abort). The differences that you point out seem to be in the > implementation, while the contract of the interface seems to be the > same. > Following your argument it seems that we should have a new > "Comparable" interface for every type that we want to compare even > tough the interface provides a contract for the compareTo method > only. > And I would also think that it’s not a bad sign, if an index > component implements the same interface as the index that it is a > part of. I guess that there's not such a big difference between a > single-component index and an index component. I think now we agree on that adding the implementation of LSMDiskComponentBulkLoader is fine (since it simplifies/unifies operations like flush/merge/index bulk load), but the question is that whether we need a new interface ILSMDiskComponentBulkLoader. Without this interface, the implementations should implement IIndexBulkLoader. For this question, the main point is that whether a disk component should be treated as an index. Personally, I think the answer is no, and the current interface hierarchy of LSMIndex/Index is also separate from LSMComponent. Making LSMComponentBulkLoader implement IIndexBulkLoader would make the code a little bit confusing. Ideally, I think there should be an interface called IBulkLoader, which is used to bulk load data into some target. IIndexBulkLoader and ILSMDiskComponentBulkLoader should be two sub interfaces of this IBulkLoader sitting in parallel. Hope this explains the motivation of this patch. -- To view, visit https://asterix-gerrit.ics.uci.edu/1773 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb Gerrit-PatchSet: 5 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: No
Change in asterixdb[master]: Add LSMDiskComponentBulkLoader
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1773 to look at the new patch set (#6). Change subject: Add LSMDiskComponentBulkLoader .. Add LSMDiskComponentBulkLoader -Added ILSMDiskComponentBulkLoader interface, which is used to bulk load an LSMDiskComponent with anti-matters -Added ILSMDiskComponentWithBuddyBTreeBulkLoader, which is used to bulk load an LSMDiskComponent with deleted-keys btrees -Refactored LSM flush/merge/index bulk load operations to use the LSMDiskComponentBulkLoader Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb --- M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentWithBuddyBTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBTreeBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader.java 14 files changed, 908 insertions(+), 529 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/73/1773/6 -- To view, visit https://asterix-gerrit.ics.uci.edu/1773 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newpatchset Gerrit-Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb Gerrit-PatchSet: 6 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi
Change in asterixdb[master]: Add LSMDiskComponentBulkLoader
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1773 to look at the new patch set (#7). Change subject: Add LSMDiskComponentBulkLoader .. Add LSMDiskComponentBulkLoader -Added ILSMDiskComponentBulkLoader interface, which is used to bulk load an LSMDiskComponent with anti-matters -Added ILSMDiskComponentWithBuddyBTreeBulkLoader, which is used to bulk load an LSMDiskComponent with deleted-keys btrees -Refactored LSM flush/merge/index bulk load operations to use the LSMDiskComponentBulkLoader Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb --- M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentWithBuddyBTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBTreeBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader.java 14 files changed, 911 insertions(+), 529 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/73/1773/7 -- To view, visit https://asterix-gerrit.ics.uci.edu/1773 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newpatchset Gerrit-Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb Gerrit-PatchSet: 7 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi
Change in asterixdb[master]: Add LSMDiskComponentBulkLoader
Luo Chen has posted comments on this change. Change subject: Add LSMDiskComponentBulkLoader .. Patch Set 7: > Chen Luo, > I don't think the differences are fundamental enough to warrant > creating a new interface. can we keep using the IIndexBulkLoader > instead of the ILSMDiskComponentBulkLoader? We can keep the > ILSMDiskComponentWithBuddyBTreeBulkLoader which would extend the > IIndexBulkLoader. everything else should stay the same IMO. > > Thoughts? It's OK to me. I'll make necessary changes and submit a new patch tomorrow. -- To view, visit https://asterix-gerrit.ics.uci.edu/1773 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb Gerrit-PatchSet: 7 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: No
Change in asterixdb[master]: Add LSMDiskComponentBulkLoader
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1773 to look at the new patch set (#8). Change subject: Add LSMDiskComponentBulkLoader .. Add LSMDiskComponentBulkLoader -Added ILSMDiskComponentBulkLoader interface, which is used to bulk load an LSMDiskComponent with anti-matters -Added ILSMDiskComponentWithBuddyBTreeBulkLoader, which is used to bulk load an LSMDiskComponent with deleted-keys btrees -Refactored LSM flush/merge/index bulk load operations to use the LSMDiskComponentBulkLoader Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb --- M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader.java 16 files changed, 956 insertions(+), 688 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/73/1773/8 -- To view, visit https://asterix-gerrit.ics.uci.edu/1773 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newpatchset Gerrit-Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb Gerrit-PatchSet: 8 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi
Change in asterixdb[master]: Add LSMDiskComponentBulkLoader
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1773 to look at the new patch set (#9). Change subject: Add LSMDiskComponentBulkLoader .. Add LSMDiskComponentBulkLoader -Added LSMDiskComponentBulkLoader implementations, which are used to bulk load an LSMDiskComponent with anti-matters -Added LSMDiskComponentWithBuddyBTreeBulkLoader implementations, which are used to bulk load an LSMDiskComponent with deleted-keys btrees -Refactored LSM flush/merge/index bulk load operations to use the LSMDiskComponentBulkLoader Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb --- M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader.java 16 files changed, 956 insertions(+), 688 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/73/1773/9 -- To view, visit https://asterix-gerrit.ics.uci.edu/1773 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newpatchset Gerrit-Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb Gerrit-PatchSet: 9 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi
Change in asterixdb[master]: Add LSMDiskComponentBulkLoader
Luo Chen has posted comments on this change. Change subject: Add LSMDiskComponentBulkLoader .. Patch Set 9: > Chen Luo, > I don't think the differences are fundamental enough to warrant > creating a new interface. can we keep using the IIndexBulkLoader > instead of the ILSMDiskComponentBulkLoader? We can keep the > ILSMDiskComponentWithBuddyBTreeBulkLoader which would extend the > IIndexBulkLoader. everything else should stay the same IMO. > > Thoughts? Done. I removed the added new interfaces (also ILSMDiskComponentWithBuddyBulkLoader, since only keeping this interface seems a little bit strange). -- To view, visit https://asterix-gerrit.ics.uci.edu/1773 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb Gerrit-PatchSet: 9 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: No
Change in asterixdb[master]: Add LSMDiskComponentBulkLoader
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1773 to look at the new patch set (#10). Change subject: Add LSMDiskComponentBulkLoader .. Add LSMDiskComponentBulkLoader -Added LSMDiskComponentBulkLoader implementations, which are used to bulk load an LSMDiskComponent with anti-matters -Added LSMDiskComponentWithBuddyBTreeBulkLoader implementations, which are used to bulk load an LSMDiskComponent with deleted-keys btrees -Refactored LSM flush/merge/index bulk load operations to use the LSMDiskComponentBulkLoader Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb --- M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader.java 16 files changed, 957 insertions(+), 688 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/73/1773/10 -- To view, visit https://asterix-gerrit.ics.uci.edu/1773 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newpatchset Gerrit-Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb Gerrit-PatchSet: 10 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi
Change in asterixdb[master]: Add Disk Component Scan operation for primary LSM index
Luo Chen has uploaded a new change for review. https://asterix-gerrit.ics.uci.edu/1791 Change subject: Add Disk Component Scan operation for primary LSM index .. Add Disk Component Scan operation for primary LSM index -Added disk component scan operation for primary LSMBTree index, which would be used by creating new secondary index -This operation scans all disk components of the primary index, and return all tuples. Thus, tuples with the same primary key in different in components would be returned separately. -The returned tuple has an extra int field, which indicates which component this tuple comes from, and a boolean flag, which indicates whether this tuple is an anti-matter tuple or not. Change-Id: I31b2c67c58cb0a440c1d2c26400af322e2f1c1e5 --- M hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java M hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties M hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/BooleanPointable.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/ophelpers/IndexOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentScanCursor.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeSearchCursor.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/LSMOperationType.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java A hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeScanDiskComponentsTest.java 17 files changed, 622 insertions(+), 5 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/91/1791/1 diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java index 23031bc..d0c6f60 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java @@ -89,6 +89,7 @@ public static final int CANNOT_CLEAR_INACTIVE_INDEX = 53; public static final int CANNOT_ALLOCATE_MEMORY_FOR_INACTIVE_INDEX = 54; public static final int RESOURCE_DOES_NOT_EXIST = 55; +public static final int DISK_COMPONENT_SCAN_NOT_SUPPORTED_IN_SECONDARY_INDEX = 56; // Compilation error codes. public static final int RULECOLLECTION_NOT_INSTANCE_OF_LIST = 1; diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties index 88e4204..3dbc926 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties @@ -74,5 +74,6 @@ 53 = Failed to clear the index since it is inactive 54 = Failed to allocate memory components for the index since it is inactive 55 = Resource does not exist for %1$s +56 = LSM disk component scan is not supported in Secondary LSM Index 1 = The given rule collection %1$s is not an instance of the List class. diff --git a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/jav
Change in asterixdb[master]: Add LSMDiskComponentBulkLoader
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1773 to look at the new patch set (#11). Change subject: Add LSMDiskComponentBulkLoader .. Add LSMDiskComponentBulkLoader -Added LSMDiskComponentBulkLoader implementations, which are used to bulk load an LSMDiskComponent with anti-matters -Added LSMDiskComponentWithBuddyBTreeBulkLoader implementations, which are used to bulk load an LSMDiskComponent with deleted-keys btrees -Refactored LSM flush/merge/index bulk load operations to use the LSMDiskComponentBulkLoader Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb --- M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader.java 16 files changed, 957 insertions(+), 688 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/73/1773/11 -- To view, visit https://asterix-gerrit.ics.uci.edu/1773 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newpatchset Gerrit-Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb Gerrit-PatchSet: 11 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi
Change in asterixdb[master]: Add LSMDiskComponentBulkLoader
Luo Chen has posted comments on this change. Change subject: Add LSMDiskComponentBulkLoader .. Patch Set 11: (1 comment) > (1 comment) > > One last comment. otherwise looks good. > Will +2 once this is addressed https://asterix-gerrit.ics.uci.edu/#/c/1773/11/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java File hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java: PS11, Line 156: boolean hasBloomFilter > this is a property of the ILSMIndex and so, there seems to be no need to pa There is only one exception to this. Consider LSMRTree (with the buddy btree). Normally, its disk component has a buddy btree and the associated bloom filter. However, when the merge operation produces the oldest disk component, its buddy btree is empty and there is no associated bloom filter (through we have to create an empty buddy-btree, otherwise this disk component is considered invalid by the current system). -- To view, visit https://asterix-gerrit.ics.uci.edu/1773 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb Gerrit-PatchSet: 11 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: Yes
Change in asterixdb[master]: Implemented Disk Components Alignment Based on IDs
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1761 to look at the new patch set (#11). Change subject: Implemented Disk Components Alignment Based on IDs .. Implemented Disk Components Alignment Based on IDs - Added IDs (using LSN) for disk components. When a disk component is flushed, its initial ID is set as the LSN. When components are merged, the result ID is the union of all IDs to be merged. - Changed the correlated merge policy to correlate disk components of the primary and secondaries using IDs. Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e --- M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetResource.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/IndexInfo.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java A asterixdb/asterix-common/src/test/java/org/apache/asterix/test/context/CorrelatedPrefixMergePolicyTest.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentId.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMDiskComponentId.java 11 files changed, 750 insertions(+), 59 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/61/1761/11 -- To view, visit https://asterix-gerrit.ics.uci.edu/1761 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newpatchset Gerrit-Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e Gerrit-PatchSet: 11 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi
Change in asterixdb[master]: Implemented Disk Components Alignment Based on IDs
Luo Chen has posted comments on this change. Change subject: Implemented Disk Components Alignment Based on IDs .. Patch Set 11: (1 comment) > (2 comments) > > a very small comment. and the old comment I thought I posted > before. > This is very nice work (+1). will +2 once the comment is addressed https://asterix-gerrit.ics.uci.edu/#/c/1761/10/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/IndexInfo.java File asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/IndexInfo.java: PS10, Line 26: resourceId > why rename this? seems like we are moving towards having Id instead of ID : This is changed by accident... I reverted back. -- To view, visit https://asterix-gerrit.ics.uci.edu/1761 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e Gerrit-PatchSet: 11 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: Yes
Change in asterixdb[master]: Add LSMDiskComponentBulkLoader
Luo Chen has posted comments on this change. Change subject: Add LSMDiskComponentBulkLoader .. Patch Set 11: > (1 comment) Yes it should be the case, and (hopefully) it would simply create an empty bloom filter on disk. -- To view, visit https://asterix-gerrit.ics.uci.edu/1773 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb Gerrit-PatchSet: 11 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: No
Change in asterixdb[master]: Add LSMDiskComponentBulkLoader
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1773 to look at the new patch set (#12). Change subject: Add LSMDiskComponentBulkLoader .. Add LSMDiskComponentBulkLoader -Added LSMDiskComponentBulkLoader implementations, which are used to bulk load an LSMDiskComponent with anti-matters -Added LSMDiskComponentWithBuddyBTreeBulkLoader implementations, which are used to bulk load an LSMDiskComponent with deleted-keys btrees -Refactored LSM flush/merge/index bulk load operations to use the LSMDiskComponentBulkLoader Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb --- M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader.java 16 files changed, 952 insertions(+), 688 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/73/1773/12 -- To view, visit https://asterix-gerrit.ics.uci.edu/1773 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newpatchset Gerrit-Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb Gerrit-PatchSet: 12 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi
Change in asterixdb[master]: Add LSMDiskComponentBulkLoader
Luo Chen has posted comments on this change. Change subject: Add LSMDiskComponentBulkLoader .. Patch Set 12: > (1 comment) Hi, I submitted a new patch to remove the hasBloomFilter parameter. However, to be consistent with the current behavior, the component bulk loader creates the bloom filter only when numElementsHint>0 (numElements has to be known in advance in order to create the bloom filter, otherwise, it implies the bloom filter is not needed). -- To view, visit https://asterix-gerrit.ics.uci.edu/1773 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I772a6d68761fcbb85982a1c9f72f2d186e1d1ffb Gerrit-PatchSet: 12 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: No
Change in asterixdb[master]: Implemented Disk Components Alignment Based on IDs
Luo Chen has posted comments on this change. Change subject: Implemented Disk Components Alignment Based on IDs .. Patch Set 11: Hi Abdullah, I still don't have privilege to submit the patch. Could you help to submit the two approved patches? Thanks! -- To view, visit https://asterix-gerrit.ics.uci.edu/1761 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I768ee9ac0a8d3c99c631086093a6b778b2e7588e Gerrit-PatchSet: 11 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Ian Maxon Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Jianfeng Jia Gerrit-Reviewer: Luo Chen Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: Yingyi Bu Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: No
Change in asterixdb[master]: Allow change insert/delete mode for anti-matter LSM index
Luo Chen has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/1796
Change subject: Allow change insert/delete mode for anti-matter LSM index
..
Allow change insert/delete mode for anti-matter LSM index
Currently, when we bulk load an lsm index, the leaf factory of the index
only allows to insert tuples, but not deleting tuples (i.e.,
inserting anti-matter tuples). However, the new change of creating
secondary index requires inserting both matter and anti-matter tuples.
This change exposes set insert/delete mode API for tuple writer factory
when we bulk load an lsm index component. This is achived basically
the same way as bulk loading an ExternalBTree.
Change-Id: I6665f56a5d2183697197298fa24824eeb827686a
---
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
D
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeRefrencingTupleWriterFactory.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriter.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriterFactory.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
A
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTreeTupleWriter.java
A
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTreeTupleWriterFactory.java
A
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/tuples/LSMTreeRefrencingTupleWriterFactory.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriter.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactory.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactoryForPointMBR.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterForPointMBR.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
13 files changed, 231 insertions(+), 124 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/96/1796/1
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
index c641dc1..3f92a3c 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
@@ -35,11 +35,9 @@
import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
import org.apache.hyracks.storage.am.common.api.ITwoPCIndexBulkLoader;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
-import
org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeRefrencingTupleWriterFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
@@ -50,6 +48,7 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriterFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ITwoPCIndex;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
import
org.apache.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
@@ -449,7 +448,7 @@
public class LSMTwoPCBTreeBulkLoader implements IIndexBulkLoader,
ITwoPCIndexBulkLoader {
privat
Change in asterixdb[master]: Allow change insert/delete mode for anti-matter LSM index
Hello Jenkins, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/1796 to look at the new patch set (#2). Change subject: Allow change insert/delete mode for anti-matter LSM index .. Allow change insert/delete mode for anti-matter LSM index Currently, when we bulk load an lsm index, the leaf factory of the index only allows to insert tuples, but not deleting tuples (i.e., inserting anti-matter tuples). However, the new change of creating secondary index requires inserting both matter and anti-matter tuples. This change exposes set insert/delete mode API for tuple writer factory when we bulk load an lsm index component. This is achived basically the same way as bulk loading an ExternalBTree. Change-Id: I6665f56a5d2183697197298fa24824eeb827686a --- M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java D hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeRefrencingTupleWriterFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriter.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriterFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTreeTupleWriter.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/tuples/LSMTreeRefrencingTupleWriterFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriter.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactoryForPointMBR.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterForPointMBR.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java 20 files changed, 275 insertions(+), 146 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/96/1796/2 -- To view, visit https://asterix-gerrit.ics.uci.edu/1796 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newpatchset Gerrit-Change-Id: I6665f56a5d2183697197298fa24824eeb827686a Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen Gerrit-Reviewer: Jenkins
