Repository: asterixdb Updated Branches: refs/heads/master 46518babd -> d26141d6b
Allow insert anti-matter tuples when bulk loading LSM index Previously, when we bulk load an LSM index, we are not allowed to insert anti-matter tuples to the disk component. However, creating secondary index for correlated datasets requires anti-matter tuples to be inserted as well. Thus, this patch mainly contains the following changes: - When bulk loading LSM index, allow the user to switch between insert mode and delete mode - Extended the LSMDiskComponentBulkLoader with the delete method. For LSM index with anti-matter tuples, the delete method simply sets the TupleWriter to delete mode, and inserts the anti-matter tuple. For LSM index with buddy btree, it simply inserts the deleted tuple into the buddy btree. - Since the LSMDiskComponentBulkLoader would have a delete method anyway, added a new ILSMDiskComponentBulkLoader interface containing the delete method. Change-Id: I6665f56a5d2183697197298fa24824eeb827686a Reviewed-on: https://asterix-gerrit.ics.uci.edu/1796 Sonar-Qube: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> BAD: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: Yingyi Bu <buyin...@gmail.com> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/d26141d6 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/d26141d6 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/d26141d6 Branch: refs/heads/master Commit: d26141d6b1c09783350e5e1906d5e71bc10071f0 Parents: 46518ba Author: luochen01 <cl...@uci.edu> Authored: Fri Jun 2 22:28:15 2017 -0700 Committer: Yingyi Bu <buyin...@gmail.com> Committed: Sat Jun 3 08:45:35 2017 -0700 ---------------------------------------------------------------------- .../am/lsm/btree/impls/ExternalBTree.java | 13 +--- .../lsm/btree/impls/ExternalBTreeWithBuddy.java | 3 +- .../storage/am/lsm/btree/impls/LSMBTree.java | 7 +- .../LSMBTreeRefrencingTupleWriterFactory.java | 69 -------------------- .../lsm/btree/tuples/LSMBTreeTupleWriter.java | 13 ++-- .../tuples/LSMBTreeTupleWriterFactory.java | 13 ++-- .../am/lsm/btree/utils/LSMBTreeUtil.java | 19 ++++-- .../common/api/ILSMDiskComponentBulkLoader.java | 37 +++++++++++ .../storage/am/lsm/common/api/ILSMIndex.java | 6 +- .../am/lsm/common/api/ILSMTreeTupleWriter.java | 34 ++++++++++ .../AbstractLSMDiskComponentBulkLoader.java | 43 ++++++++++-- ...ractLSMDiskComponentWithBuddyBulkLoader.java | 20 ++++-- .../invertedindex/impls/LSMInvertedIndex.java | 9 +-- .../storage/am/lsm/rtree/impls/LSMRTree.java | 11 ++-- .../impls/LSMRTreeWithAntiMatterTuples.java | 9 ++- .../lsm/rtree/tuples/LSMRTreeTupleWriter.java | 14 ++-- .../tuples/LSMRTreeTupleWriterFactory.java | 13 ++-- .../LSMRTreeTupleWriterFactoryForPointMBR.java | 12 ++-- .../tuples/LSMRTreeTupleWriterForPointMBR.java | 25 +++++-- .../tuples/LSMTypeAwareTupleWriterFactory.java | 8 +-- .../am/lsm/rtree/utils/LSMRTreeUtils.java | 18 +++-- 21 files changed, 241 insertions(+), 155 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d26141d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java index c641dc1..a11b835 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java @@ -35,13 +35,12 @@ import org.apache.hyracks.storage.am.common.api.IIndexOperationContext; import org.apache.hyracks.storage.am.common.api.IMetadataPageManager; import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; -import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory; import org.apache.hyracks.storage.am.common.api.ITwoPCIndexBulkLoader; import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback; import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; -import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeRefrencingTupleWriterFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler; @@ -448,8 +447,7 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex { // modifications public class LSMTwoPCBTreeBulkLoader implements IIndexBulkLoader, ITwoPCIndexBulkLoader { private final ILSMDiskComponent component; - private final IIndexBulkLoader componentBulkLoader; - private final ITreeIndexTupleWriterFactory frameTupleWriterFactory; + private final ILSMDiskComponentBulkLoader componentBulkLoader; private final boolean isTransaction; @@ -463,9 +461,6 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex { component = createBulkLoadTarget(); } - frameTupleWriterFactory = - ((LSMBTreeDiskComponent) component).getBTree().getLeafFrameFactory().getTupleWriterFactory(); - componentBulkLoader = createComponentBulkLoader(component, fillFactor, verifyInput, numElementsHint, false, true); } @@ -499,9 +494,7 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex { // calling delete @Override public void delete(ITupleReference tuple) throws HyracksDataException { - ((LSMBTreeRefrencingTupleWriterFactory) frameTupleWriterFactory).setMode(IndexOperation.DELETE); - componentBulkLoader.add(tuple); - ((LSMBTreeRefrencingTupleWriterFactory) frameTupleWriterFactory).setMode(IndexOperation.INSERT); + componentBulkLoader.delete(tuple); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d26141d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java index dfa08d6..abf3fa7 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java @@ -46,6 +46,7 @@ import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback; import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; @@ -275,7 +276,7 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd } @Override - public IIndexBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor, + public ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter) throws HyracksDataException { BloomFilterSpecification bloomFilterSpec = null; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d26141d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java index a4c67c2..b224e87 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java @@ -48,6 +48,7 @@ import org.apache.hyracks.storage.am.lsm.common.api.IComponentFilterHelper; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler; @@ -311,7 +312,7 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { LSMBTreeDiskComponent component = createDiskComponent(componentFactory, flushOp.getTarget(), flushOp.getBloomFilterTarget(), true); - IIndexBulkLoader componentBulkLoader = + ILSMDiskComponentBulkLoader componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, numElements, false, false); IIndexCursor scanCursor = accessor.createSearchCursor(false); @@ -365,7 +366,7 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { LSMBTreeDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(), mergeOp.getBloomFilterTarget(), true); - IIndexBulkLoader componentBulkLoader = + ILSMDiskComponentBulkLoader componentBulkLoader = createComponentBulkLoader(mergedComponent, 1.0f, false, numElements, false, false); try { while (cursor.hasNext()) { @@ -415,7 +416,7 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { } @Override - public IIndexBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor, + public ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter) throws HyracksDataException { BloomFilterSpecification bloomFilterSpec = null; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d26141d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeRefrencingTupleWriterFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeRefrencingTupleWriterFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeRefrencingTupleWriterFactory.java deleted file mode 100644 index 24ae68c..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeRefrencingTupleWriterFactory.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hyracks.storage.am.lsm.btree.tuples; - -import org.apache.hyracks.api.dataflow.value.ITypeTraits; -import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter; -import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; -import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory; - -/** - * This is a TupleWriterFactory that can be used to reference the last writer it created - * and switch between actual and antimatter writer modes - * @author alamouda - * - */ -public class LSMBTreeRefrencingTupleWriterFactory extends TypeAwareTupleWriterFactory{ - private static final long serialVersionUID = 1L; - private final ITypeTraits[] typeTraits; - private final int numKeyFields; - private boolean isDelete; - private LSMBTreeTupleWriter createdTupleWriter; - - public LSMBTreeRefrencingTupleWriterFactory(ITypeTraits[] typeTraits, int numKeyFields, boolean isDelete) { - super(typeTraits); - this.typeTraits = typeTraits; - this.numKeyFields = numKeyFields; - this.isDelete = isDelete; - } - - @Override - public ITreeIndexTupleWriter createTupleWriter() { - createdTupleWriter = new LSMBTreeTupleWriter(typeTraits, numKeyFields, isDelete); - return createdTupleWriter; - } - - public void setMode(IndexOperation op){ - if(op == IndexOperation.INSERT){ - this.isDelete = false; - if(createdTupleWriter != null){ - this.createdTupleWriter.setAntimatter(false); - } - } else if(op == IndexOperation.DELETE){ - this.isDelete = true; - if(createdTupleWriter != null){ - this.createdTupleWriter.setAntimatter(true); - } - } - } - - public LSMBTreeTupleWriter getCreatedTupleWriter() { - return createdTupleWriter; - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d26141d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriter.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriter.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriter.java index 3c60f15..502e43b 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriter.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriter.java @@ -23,8 +23,9 @@ import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference; import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriter; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriter; -public class LSMBTreeTupleWriter extends TypeAwareTupleWriter { +public class LSMBTreeTupleWriter extends TypeAwareTupleWriter implements ILSMTreeTupleWriter { private boolean isAntimatter; private final int numKeyFields; @@ -57,13 +58,13 @@ public class LSMBTreeTupleWriter extends TypeAwareTupleWriter { @Override protected int getNullFlagsBytes(int numFields) { // +1.0 is for matter/antimatter bit. - return (int) Math.ceil(((double) numFields + 1.0) / 8.0); + return (int) Math.ceil((numFields + 1.0) / 8.0); } @Override protected int getNullFlagsBytes(ITupleReference tuple) { // +1.0 is for matter/antimatter bit. - return (int) Math.ceil(((double) tuple.getFieldCount() + 1.0) / 8.0); + return (int) Math.ceil((tuple.getFieldCount() + 1.0) / 8.0); } @Override @@ -83,8 +84,8 @@ public class LSMBTreeTupleWriter extends TypeAwareTupleWriter { targetBuf[targetOff] = (byte) (targetBuf[targetOff] | (1 << 7)); } - // Allow using the same writer for both delete and insert tuples - public void setAntimatter(boolean isAntimatter) { - this.isAntimatter = isAntimatter; + @Override + public void setAntimatter(boolean isDelete) { + this.isAntimatter = isDelete; } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d26141d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriterFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriterFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriterFactory.java index 5ee2b81..4ea501a 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriterFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriterFactory.java @@ -20,23 +20,24 @@ package org.apache.hyracks.storage.am.lsm.btree.tuples; import org.apache.hyracks.api.dataflow.value.ITypeTraits; -import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter; import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriter; public class LSMBTreeTupleWriterFactory extends TypeAwareTupleWriterFactory { private static final long serialVersionUID = 1L; private final int numKeyFields; - private final boolean isDelete; + private final boolean isAntimatter; - public LSMBTreeTupleWriterFactory(ITypeTraits[] typeTraits, int numKeyFields, boolean isDelete) { + public LSMBTreeTupleWriterFactory(ITypeTraits[] typeTraits, int numKeyFields, boolean isAntimatter) { super(typeTraits); this.numKeyFields = numKeyFields; - this.isDelete = isDelete; + this.isAntimatter = isAntimatter; } @Override - public ITreeIndexTupleWriter createTupleWriter() { - return new LSMBTreeTupleWriter(typeTraits, numKeyFields, isDelete); + public ILSMTreeTupleWriter createTupleWriter() { + return new LSMBTreeTupleWriter(typeTraits, numKeyFields, isAntimatter); } + } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d26141d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java index 0e5fcdb..259f25f 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java @@ -39,7 +39,6 @@ import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree; import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeFileManager; import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBuddyFileManager; import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeCopyTupleWriterFactory; -import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeRefrencingTupleWriterFactory; import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleWriterFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler; @@ -74,16 +73,21 @@ public class LSMBTreeUtil { new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, true); LSMBTreeCopyTupleWriterFactory copyTupleWriterFactory = new LSMBTreeCopyTupleWriterFactory(typeTraits, cmpFactories.length); + LSMBTreeTupleWriterFactory bulkLoadTupleWriterFactory = + new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false); + ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory); ITreeIndexFrameFactory copyTupleLeafFrameFactory = new BTreeNSMLeafFrameFactory(copyTupleWriterFactory); ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory); ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory); + ITreeIndexFrameFactory bulkLoadLeafFrameFactory = new BTreeNSMLeafFrameFactory(bulkLoadTupleWriterFactory); TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length); - TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, - freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length); + TreeIndexFactory<BTree> bulkLoadBTreeFactory = + new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, + interiorFrameFactory, bulkLoadLeafFrameFactory, cmpFactories, typeTraits.length); BloomFilterFactory bloomFilterFactory = needKeyDupCheck ? new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields) : null; @@ -125,11 +129,12 @@ public class LSMBTreeUtil { ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory); ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory); // This is the tuple writer that can do both inserts and deletes - LSMBTreeRefrencingTupleWriterFactory referencingTupleWriterFactory = - new LSMBTreeRefrencingTupleWriterFactory(typeTraits, cmpFactories.length, false); + LSMBTreeTupleWriterFactory transactionTupleWriterFactory = + new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false); // This is the leaf frame factory for transaction components since it // can be used for both inserts and deletes - ITreeIndexFrameFactory dualLeafFrameFactory = new BTreeNSMLeafFrameFactory(referencingTupleWriterFactory); + ITreeIndexFrameFactory transactionLeafFrameFactory = + new BTreeNSMLeafFrameFactory(transactionTupleWriterFactory); TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, @@ -143,7 +148,7 @@ public class LSMBTreeUtil { // This is the component factory for transactions TreeIndexFactory<BTree> transactionBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, - interiorFrameFactory, dualLeafFrameFactory, cmpFactories, typeTraits.length); + interiorFrameFactory, transactionLeafFrameFactory, cmpFactories, typeTraits.length); //TODO remove BloomFilter from external dataset's secondary LSMBTree index ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(ioManager, diskFileMapProvider, file, diskBTreeFactory, true); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d26141d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java new file mode 100644 index 0000000..f1d1ce1 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hyracks.storage.am.lsm.common.api; + +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; +import org.apache.hyracks.storage.common.IIndexBulkLoader; + +public interface ILSMDiskComponentBulkLoader extends IIndexBulkLoader { + + /** + * Delete the given tuple (insert an antimatter tuple or deleted-key tuple) + * from the disk component + * + * @param tuple + * @throws HyracksDataException + */ + void delete(ITupleReference tuple) throws HyracksDataException; + +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d26141d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java index 5a6f391..5b3872c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java @@ -27,7 +27,6 @@ import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; import org.apache.hyracks.storage.am.common.api.IIndexOperationContext; import org.apache.hyracks.storage.am.lsm.common.impls.LSMHarness; import org.apache.hyracks.storage.common.IIndex; -import org.apache.hyracks.storage.common.IIndexBulkLoader; import org.apache.hyracks.storage.common.IIndexCursor; import org.apache.hyracks.storage.common.IModificationOperationCallback; import org.apache.hyracks.storage.common.ISearchOperationCallback; @@ -153,7 +152,8 @@ public interface ILSMIndex extends IIndex { * @return * @throws HyracksDataException */ - IIndexBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor, boolean verifyInput, - long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter) throws HyracksDataException; + ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor, + boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter) + throws HyracksDataException; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d26141d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTreeTupleWriter.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTreeTupleWriter.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTreeTupleWriter.java new file mode 100644 index 0000000..1d6201d --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTreeTupleWriter.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hyracks.storage.am.lsm.common.api; + +import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter; + +/** + * This interface allows to write both insert (matter) and delete (anti-matter) tuples + * + * @author luochen + * + */ +public interface ILSMTreeTupleWriter extends ITreeIndexTupleWriter { + + void setAntimatter(boolean isAntimatter); + +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d26141d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java index 964893a..ad47bbc 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java @@ -23,14 +23,17 @@ import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter; import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification; import org.apache.hyracks.storage.am.common.api.ITreeIndex; +import org.apache.hyracks.storage.am.common.impls.AbstractTreeIndex.AbstractTreeIndexBulkLoader; import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterManager; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriter; import org.apache.hyracks.storage.common.IIndex; import org.apache.hyracks.storage.common.IIndexBulkLoader; import org.apache.hyracks.storage.common.MultiComparator; -public abstract class AbstractLSMDiskComponentBulkLoader implements IIndexBulkLoader { +public abstract class AbstractLSMDiskComponentBulkLoader implements ILSMDiskComponentBulkLoader { protected final ILSMDiskComponent component; protected final IIndexBulkLoader indexBulkLoader; @@ -87,14 +90,39 @@ public abstract class AbstractLSMDiskComponentBulkLoader implements IIndexBulkLo if (bloomFilterBuilder != null) { bloomFilterBuilder.add(t); } + updateFilter(tuple); - if (filterTuple != null) { - filterTuple.reset(tuple); - component.getLSMComponentFilter().update(filterTuple, filterCmp); + } catch (Exception e) { + cleanupArtifacts(); + throw e; + } + if (isEmptyComponent) { + isEmptyComponent = false; + } + } + + @Override + public void delete(ITupleReference tuple) throws HyracksDataException { + ILSMTreeTupleWriter tupleWriter = + (ILSMTreeTupleWriter) ((AbstractTreeIndexBulkLoader) indexBulkLoader).getLeafFrame().getTupleWriter(); + tupleWriter.setAntimatter(true); + try { + ITupleReference t; + if (indexTuple != null) { + indexTuple.reset(tuple); + t = indexTuple; + } else { + t = tuple; } + + indexBulkLoader.add(t); + + updateFilter(tuple); } catch (Exception e) { cleanupArtifacts(); throw e; + } finally { + tupleWriter.setAntimatter(false); } if (isEmptyComponent) { isEmptyComponent = false; @@ -148,6 +176,13 @@ public abstract class AbstractLSMDiskComponentBulkLoader implements IIndexBulkLo } } + protected void updateFilter(ITupleReference tuple) throws HyracksDataException { + if (filterTuple != null) { + filterTuple.reset(tuple); + component.getLSMComponentFilter().update(filterTuple, filterCmp); + } + } + /** * TreeIndex is used to hold the filter tuple values * http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d26141d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBulkLoader.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBulkLoader.java index 453d6cf..d3eafdc 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBulkLoader.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBulkLoader.java @@ -59,10 +59,7 @@ public abstract class AbstractLSMDiskComponentWithBuddyBulkLoader extends Abstra indexBulkLoader.add(t); - if (filterTuple != null) { - filterTuple.reset(tuple); - component.getLSMComponentFilter().update(filterTuple, filterCmp); - } + updateFilter(tuple); } catch (Exception e) { cleanupArtifacts(); throw e; @@ -72,12 +69,23 @@ public abstract class AbstractLSMDiskComponentWithBuddyBulkLoader extends Abstra } } + @Override public void delete(ITupleReference tuple) throws HyracksDataException { try { - buddyBTreeBulkLoader.add(tuple); + ITupleReference t; + if (indexTuple != null) { + indexTuple.reset(tuple); + t = indexTuple; + } else { + t = tuple; + } + + buddyBTreeBulkLoader.add(t); if (bloomFilterBuilder != null) { - bloomFilterBuilder.add(tuple); + bloomFilterBuilder.add(t); } + + updateFilter(tuple); } catch (HyracksDataException e) { //deleting a key multiple times is OK if (e.getErrorCode() != ErrorCode.DUPLICATE_KEY) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d26141d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java index f827b21..bd4797a 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java @@ -49,6 +49,7 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; @@ -362,7 +363,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex btreeCountingCursor.close(); } - IIndexBulkLoader componentBulkLoader = + ILSMDiskComponentBulkLoader componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, numBTreeTuples, false, false); // Create a scan cursor on the deleted keys BTree underlying the in-memory inverted index. @@ -425,7 +426,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex LSMInvertedIndexDiskComponent component = createDiskInvIndexComponent(componentFactory, mergeOp.getTarget(), mergeOp.getDeletedKeysBTreeTarget(), mergeOp.getBloomFilterTarget(), true); - IIndexBulkLoader componentBulkLoader; + ILSMDiskComponentBulkLoader componentBulkLoader; // In case we must keep the deleted-keys BTrees, then they must be merged *before* merging the inverted indexes so that // lsmHarness.endSearch() is called once when the inverted indexes have been merged. @@ -448,7 +449,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex while (btreeCursor.hasNext()) { btreeCursor.next(); ITupleReference tuple = btreeCursor.getTuple(); - ((LSMInvertedIndexDiskComponentBulkLoader) componentBulkLoader).delete(tuple); + componentBulkLoader.delete(tuple); } } finally { btreeCursor.close(); @@ -489,7 +490,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex } @Override - public IIndexBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor, + public ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter) throws HyracksDataException { BloomFilterSpecification bloomFilterSpec = null; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d26141d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java index 3618737..eb838a9 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java @@ -49,6 +49,7 @@ import org.apache.hyracks.storage.am.lsm.common.api.IComponentFilterHelper; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler; @@ -195,7 +196,7 @@ public class LSMRTree extends AbstractLSMRTree { btreeCountingCursor.close(); } - IIndexBulkLoader componentBulkLoader = + ILSMDiskComponentBulkLoader componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, numBTreeTuples, false, false); ITreeIndexCursor cursor; @@ -240,7 +241,7 @@ public class LSMRTree extends AbstractLSMRTree { while (btreeScanCursor.hasNext()) { btreeScanCursor.next(); ITupleReference frameTuple = btreeScanCursor.getTuple(); - ((LSMRTreeDiskComponentBulkLoader) componentBulkLoader).delete(frameTuple); + componentBulkLoader.delete(frameTuple); } } finally { btreeScanCursor.close(); @@ -272,7 +273,7 @@ public class LSMRTree extends AbstractLSMRTree { LSMRTreeDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(), mergeOp.getBTreeTarget(), mergeOp.getBloomFilterTarget(), true); - IIndexBulkLoader componentBulkLoader; + ILSMDiskComponentBulkLoader componentBulkLoader; // In case we must keep the deleted-keys BTrees, then they must be merged *before* merging the r-trees so that // lsmHarness.endSearch() is called once when the r-trees have been merged. @@ -293,7 +294,7 @@ public class LSMRTree extends AbstractLSMRTree { while (btreeCursor.hasNext()) { btreeCursor.next(); ITupleReference tuple = btreeCursor.getTuple(); - ((LSMRTreeDiskComponentBulkLoader) componentBulkLoader).delete(tuple); + componentBulkLoader.delete(tuple); } } finally { btreeCursor.close(); @@ -343,7 +344,7 @@ public class LSMRTree extends AbstractLSMRTree { } @Override - public IIndexBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor, + public ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter) throws HyracksDataException { BloomFilterSpecification bloomFilterSpec = null; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d26141d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java index 24c46d7..81fd981 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java @@ -39,6 +39,7 @@ import org.apache.hyracks.storage.am.lsm.common.api.IComponentFilterHelper; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; @@ -135,7 +136,8 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree { SearchPredicate rtreeNullPredicate = new SearchPredicate(null, null); memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate); LSMRTreeDiskComponent component = createDiskComponent(componentFactory, flushOp.getTarget(), null, null, true); - IIndexBulkLoader componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, 0L, false, false); + ILSMDiskComponentBulkLoader componentBulkLoader = + createComponentBulkLoader(component, 1.0f, false, 0L, false, false); // Since the LSM-RTree is used as a secondary assumption, the // primary key will be the last comparator in the BTree comparators @@ -221,7 +223,8 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree { // Bulk load the tuples from all on-disk RTrees into the new RTree. LSMRTreeDiskComponent component = createDiskComponent(componentFactory, mergeOp.getTarget(), null, null, true); - IIndexBulkLoader componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, 0L, false, false); + ILSMDiskComponentBulkLoader componentBulkLoader = + createComponentBulkLoader(component, 1.0f, false, 0L, false, false); try { while (cursor.hasNext()) { cursor.next(); @@ -254,7 +257,7 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree { } @Override - public IIndexBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor, + public ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter) throws HyracksDataException { if (withFilter && filterFields != null) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d26141d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriter.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriter.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriter.java index 3e586e6..83eda02 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriter.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriter.java @@ -22,10 +22,11 @@ package org.apache.hyracks.storage.am.lsm.rtree.tuples; import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriter; import org.apache.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriter; -public class LSMRTreeTupleWriter extends RTreeTypeAwareTupleWriter { - private final boolean isAntimatter; +public class LSMRTreeTupleWriter extends RTreeTypeAwareTupleWriter implements ILSMTreeTupleWriter { + private boolean isAntimatter; public LSMRTreeTupleWriter(ITypeTraits[] typeTraits, boolean isAntimatter) { super(typeTraits); @@ -54,13 +55,13 @@ public class LSMRTreeTupleWriter extends RTreeTypeAwareTupleWriter { @Override protected int getNullFlagsBytes(int numFields) { // +1.0 is for matter/antimatter bit. - return (int) Math.ceil(((double) numFields + 1.0) / 8.0); + return (int) Math.ceil((numFields + 1.0) / 8.0); } @Override protected int getNullFlagsBytes(ITupleReference tuple) { // +1.0 is for matter/antimatter bit. - return (int) Math.ceil(((double) tuple.getFieldCount() + 1.0) / 8.0); + return (int) Math.ceil((tuple.getFieldCount() + 1.0) / 8.0); } protected void setAntimatterBit(byte[] targetBuf, int targetOff) { @@ -68,4 +69,9 @@ public class LSMRTreeTupleWriter extends RTreeTypeAwareTupleWriter { targetBuf[targetOff] = (byte) (targetBuf[targetOff] | (1 << 7)); } + @Override + public void setAntimatter(boolean isAntimatter) { + this.isAntimatter = isAntimatter; + } + } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d26141d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactory.java index 0c21194..4d06d84 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactory.java @@ -20,23 +20,24 @@ package org.apache.hyracks.storage.am.lsm.rtree.tuples; import org.apache.hyracks.api.dataflow.value.ITypeTraits; -import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter; import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriter; public class LSMRTreeTupleWriterFactory extends TypeAwareTupleWriterFactory { private static final long serialVersionUID = 1L; private final ITypeTraits[] typeTraits; - private final boolean isDelete; + private final boolean isAntimatter; - public LSMRTreeTupleWriterFactory(ITypeTraits[] typeTraits, boolean isDelete) { + public LSMRTreeTupleWriterFactory(ITypeTraits[] typeTraits, boolean isAntimatter) { super(typeTraits); this.typeTraits = typeTraits; - this.isDelete = isDelete; + this.isAntimatter = isAntimatter; } @Override - public ITreeIndexTupleWriter createTupleWriter() { - return new LSMRTreeTupleWriter(typeTraits, isDelete); + public ILSMTreeTupleWriter createTupleWriter() { + return new LSMRTreeTupleWriter(typeTraits, isAntimatter); } + } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d26141d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactoryForPointMBR.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactoryForPointMBR.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactoryForPointMBR.java index c18823a..eec6f3b 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactoryForPointMBR.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactoryForPointMBR.java @@ -20,8 +20,8 @@ package org.apache.hyracks.storage.am.lsm.rtree.tuples; import org.apache.hyracks.api.dataflow.value.ITypeTraits; -import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter; import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriter; public class LSMRTreeTupleWriterFactoryForPointMBR extends TypeAwareTupleWriterFactory { @@ -29,17 +29,21 @@ public class LSMRTreeTupleWriterFactoryForPointMBR extends TypeAwareTupleWriterF private final int keyFieldCount; private final int valueFieldCount; private final boolean antimatterAware; + private final boolean isAntimatter; public LSMRTreeTupleWriterFactoryForPointMBR(ITypeTraits[] typeTraits, int keyFieldCount, int valueFieldCount, - boolean antimatterAware) { + boolean antimatterAware, boolean isDelete) { super(typeTraits); this.keyFieldCount = keyFieldCount; this.valueFieldCount = valueFieldCount; this.antimatterAware = antimatterAware; + this.isAntimatter = isDelete; } @Override - public ITreeIndexTupleWriter createTupleWriter() { - return new LSMRTreeTupleWriterForPointMBR(typeTraits, keyFieldCount, valueFieldCount, antimatterAware); + public ILSMTreeTupleWriter createTupleWriter() { + return new LSMRTreeTupleWriterForPointMBR(typeTraits, keyFieldCount, valueFieldCount, antimatterAware, + isAntimatter); } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d26141d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterForPointMBR.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterForPointMBR.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterForPointMBR.java index 5de4d20..4ccfd47 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterForPointMBR.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterForPointMBR.java @@ -23,6 +23,7 @@ import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference; import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleReference; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriter; import org.apache.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriter; import org.apache.hyracks.util.encoding.VarLenIntEncoderDecoder; @@ -40,16 +41,17 @@ import org.apache.hyracks.util.encoding.VarLenIntEncoderDecoder; * the stored point MBR [0.4, 0.3, 1] and generates a tuple reference which is externally shown as [0.4, 0.3, 0.4, 0.3, 1]. */ -public class LSMRTreeTupleWriterForPointMBR extends RTreeTypeAwareTupleWriter { +public class LSMRTreeTupleWriterForPointMBR extends RTreeTypeAwareTupleWriter implements ILSMTreeTupleWriter { private final int inputKeyFieldCount; //double field count for mbr secondary key of an input tuple private final int valueFieldCount; //value(or payload or primary key) field count (same for an input tuple and a stored tuple) private final int inputTotalFieldCount; //total field count (key + value fields) of an input tuple. private final int storedKeyFieldCount; //double field count to be stored for the mbr secondary key private final int storedTotalFieldCount; //total field count (key + value fields) of a stored tuple. private final boolean antimatterAware; + private boolean isAntimatter; public LSMRTreeTupleWriterForPointMBR(ITypeTraits[] typeTraits, int keyFieldCount, int valueFieldCount, - boolean antimatterAware) { + boolean antimatterAware, boolean isAntimatter) { super(typeTraits); this.inputKeyFieldCount = keyFieldCount; this.valueFieldCount = valueFieldCount; @@ -57,6 +59,7 @@ public class LSMRTreeTupleWriterForPointMBR extends RTreeTypeAwareTupleWriter { this.storedKeyFieldCount = keyFieldCount / 2; this.storedTotalFieldCount = storedKeyFieldCount + valueFieldCount; this.antimatterAware = antimatterAware; + this.isAntimatter = isAntimatter; } @Override @@ -87,7 +90,7 @@ public class LSMRTreeTupleWriterForPointMBR extends RTreeTypeAwareTupleWriter { targetBuf[runner++] = (byte) 0; } - // write field slots for variable length fields which applies only to value fields in RTree + // write field slots for variable length fields which applies only to value fields in RTree for (int i = inputKeyFieldCount; i < inputTotalFieldCount; i++) { if (!typeTraits[i].isFixedLength()) { runner += VarLenIntEncoderDecoder.encode(tuple.getFieldLength(i), targetBuf, runner); @@ -106,12 +109,19 @@ public class LSMRTreeTupleWriterForPointMBR extends RTreeTypeAwareTupleWriter { } //set antimatter bit if necessary + //this is used when we flush an in-memory rtree into disk + //and insert anti-matter tuples from in-memory buddy btree into disk rtree if (antimatterAware) { if (tuple instanceof ILSMTreeTupleReference && ((ILSMTreeTupleReference) tuple).isAntimatter()) { setAntimatterBit(targetBuf, targetOff); } } + //this is used during creating secondary index operation, where we explicitly insert some antimatter tuple + if (isAntimatter) { + setAntimatterBit(targetBuf, targetOff); + } + return runner - targetOff; } @@ -123,10 +133,12 @@ public class LSMRTreeTupleWriterForPointMBR extends RTreeTypeAwareTupleWriter { "writeTupleFields(ITupleReference, int, int, byte[], int) not implemented for RTreeTypeAwareTupleWriterForPointMBR class."); } + @Override protected int getNullFlagsBytes(ITupleReference tuple) { - return (int) Math.ceil((double) (storedTotalFieldCount + (antimatterAware ? 1 : 0)) / 8.0); + return (int) Math.ceil((storedTotalFieldCount + (antimatterAware ? 1 : 0)) / 8.0); } + @Override protected int getFieldSlotsBytes(ITupleReference tuple) { int fieldSlotBytes = 0; for (int i = inputKeyFieldCount; i < inputTotalFieldCount; i++) { @@ -146,4 +158,9 @@ public class LSMRTreeTupleWriterForPointMBR extends RTreeTypeAwareTupleWriter { // Set leftmost bit to 1. targetBuf[targetOff] = (byte) (targetBuf[targetOff] | (1 << 7)); } + + @Override + public void setAntimatter(boolean isAntimatter) { + this.isAntimatter = isAntimatter; + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d26141d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMTypeAwareTupleWriterFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMTypeAwareTupleWriterFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMTypeAwareTupleWriterFactory.java index ac4f3c8..31a9278 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMTypeAwareTupleWriterFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMTypeAwareTupleWriterFactory.java @@ -28,17 +28,17 @@ public class LSMTypeAwareTupleWriterFactory extends TypeAwareTupleWriterFactory private static final long serialVersionUID = 1L; private ITypeTraits[] typeTraits; - private final boolean isDelete; + private final boolean isAntimatter; - public LSMTypeAwareTupleWriterFactory(ITypeTraits[] typeTraits, boolean isDelete) { + public LSMTypeAwareTupleWriterFactory(ITypeTraits[] typeTraits, boolean isAntimatter) { super(typeTraits); this.typeTraits = typeTraits; - this.isDelete = isDelete; + this.isAntimatter = isAntimatter; } @Override public ITreeIndexTupleWriter createTupleWriter() { - if (isDelete) { + if (isAntimatter) { return new TypeAwareTupleWriter(typeTraits); } else { return new RTreeTypeAwareTupleWriter(typeTraits); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d26141d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java index d390be0..e37fa4a 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java @@ -92,7 +92,7 @@ public class LSMRTreeUtils { ITreeIndexTupleWriterFactory rtreeLeafFrameTupleWriterFactory = null; if (isPointMBR) { rtreeLeafFrameTupleWriterFactory = - new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, false); + new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, false, false); } else { rtreeLeafFrameTupleWriterFactory = rtreeInteriorFrameTupleWriterFactory; } @@ -154,24 +154,30 @@ public class LSMRTreeUtils { new LSMRTreeTupleWriterFactory(typeTraits, false); ITreeIndexTupleWriterFactory rtreeLeafFrameTupleWriterFactory; ITreeIndexTupleWriterFactory rtreeLeafFrameCopyTupleWriterFactory; + ITreeIndexTupleWriterFactory rtreeLeafFrameBulkLoadWriterFactory; if (isPointMBR) { int keyFieldCount = rtreeCmpFactories.length; int valueFieldCount = btreeComparatorFactories.length - keyFieldCount; rtreeLeafFrameTupleWriterFactory = - new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, true); + new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, true, false); rtreeLeafFrameCopyTupleWriterFactory = - new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, true); - + new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, true, false); + rtreeLeafFrameBulkLoadWriterFactory = + new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, true, false); } else { rtreeLeafFrameTupleWriterFactory = new LSMRTreeTupleWriterFactory(typeTraits, false); rtreeLeafFrameCopyTupleWriterFactory = new LSMRTreeCopyTupleWriterFactory(typeTraits); + rtreeLeafFrameBulkLoadWriterFactory = new LSMRTreeTupleWriterFactory(typeTraits, false); } + LSMRTreeTupleWriterFactory btreeTupleWriterFactory = new LSMRTreeTupleWriterFactory(typeTraits, true); ITreeIndexFrameFactory rtreeInteriorFrameFactory = new RTreeNSMInteriorFrameFactory( rtreeInteriorFrameTupleWriterFactory, valueProviderFactories, rtreePolicyType, isPointMBR); ITreeIndexFrameFactory rtreeLeafFrameFactory = new RTreeNSMLeafFrameFactory(rtreeLeafFrameTupleWriterFactory, valueProviderFactories, rtreePolicyType, isPointMBR); + ITreeIndexFrameFactory rtreeLeafFrameBulkLoadFactory = new RTreeNSMLeafFrameFactory( + rtreeLeafFrameBulkLoadWriterFactory, valueProviderFactories, rtreePolicyType, isPointMBR); ITreeIndexFrameFactory btreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(btreeTupleWriterFactory); ITreeIndexFrameFactory btreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(btreeTupleWriterFactory); @@ -183,7 +189,7 @@ public class LSMRTreeUtils { typeTraits.length, isPointMBR); TreeIndexFactory<RTree> bulkLoadRTreeFactory = new RTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, - freePageManagerFactory, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, rtreeCmpFactories, + freePageManagerFactory, rtreeInteriorFrameFactory, rtreeLeafFrameBulkLoadFactory, rtreeCmpFactories, typeTraits.length, isPointMBR); // The first field is for the sorted curve (e.g. Hilbert curve), and the @@ -242,7 +248,7 @@ public class LSMRTreeUtils { ITreeIndexTupleWriterFactory rtreeLeafFrameTupleWriterFactory = null; if (isPointMBR) { rtreeLeafFrameTupleWriterFactory = - new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, false); + new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, false, false); } else { rtreeLeafFrameTupleWriterFactory = rtreeInteriorFrameTupleWriterFactory; }
