http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesSecondaryIndexSearchOperatorTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesSecondaryIndexSearchOperatorTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesSecondaryIndexSearchOperatorTest.java index 4d2e0a3..04179c2 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesSecondaryIndexSearchOperatorTest.java +++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesSecondaryIndexSearchOperatorTest.java @@ -19,9 +19,14 @@ package org.apache.hyracks.tests.am.lsm.rtree; +import java.io.DataOutput; + import org.apache.hyracks.api.constraints.PartitionConstraintHelper; import org.apache.hyracks.api.dataflow.IOperatorDescriptor; -import org.apache.hyracks.api.dataflow.value.*; +import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; +import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory; +import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; +import org.apache.hyracks.api.dataflow.value.RecordDescriptor; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileSplit; import org.apache.hyracks.api.job.JobSpecification; @@ -32,20 +37,17 @@ import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider; import org.apache.hyracks.dataflow.std.file.IFileSplitProvider; import org.apache.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor; import org.apache.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor; -import org.apache.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor; +import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory; import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory; -import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory; import org.apache.hyracks.storage.am.rtree.dataflow.RTreeSearchOperatorDescriptor; import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType; +import org.apache.hyracks.storage.common.IResourceFactory; import org.apache.hyracks.test.support.TestStorageManagerComponentHolder; import org.apache.hyracks.tests.am.common.ITreeIndexOperatorTestHelper; import org.apache.hyracks.tests.am.rtree.RTreeSecondaryIndexSearchOperatorTest; -import org.apache.hyracks.tests.util.NoopMissingWriterFactory; import org.junit.Test; -import java.io.DataOutput; - public class LSMRTreeWithAntiMatterTuplesSecondaryIndexSearchOperatorTest extends RTreeSecondaryIndexSearchOperatorTest { public LSMRTreeWithAntiMatterTuplesSecondaryIndexSearchOperatorTest() { @@ -58,14 +60,14 @@ public class LSMRTreeWithAntiMatterTuplesSecondaryIndexSearchOperatorTest } @Override - protected IIndexDataflowHelperFactory createDataFlowHelperFactory( + protected IResourceFactory createSecondaryResourceFactory( IPrimitiveValueProviderFactory[] secondaryValueProviderFactories, RTreePolicyType rtreePolicyType, IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory, - int[] btreeFields, int[] rtreeFields, ITypeTraits[] filterTypeTraits, - IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) { - return ((LSMRTreeWithAntiMatterTuplesOperatorTestHelper) testHelper) - .createDataFlowHelperFactory(secondaryValueProviderFactories, rtreePolicyType, btreeComparatorFactories, - linearizerCmpFactory, rtreeFields, filterTypeTraits, filterCmpFactories, filterFields); + int[] btreeFields) { + return ((LSMRTreeWithAntiMatterTuplesOperatorTestHelper) testHelper).getSecondaryLocalResourceFactory( + storageManager, secondaryValueProviderFactories, rtreePolicyType, btreeComparatorFactories, + linearizerCmpFactory, btreeFields, secondaryTypeTraits, secondaryComparatorFactories, + (IMetadataPageManagerFactory) pageManagerFactory); } @Test @@ -87,16 +89,13 @@ public class LSMRTreeWithAntiMatterTuplesSecondaryIndexSearchOperatorTest { DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE }; RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers); - ConstantTupleSourceOperatorDescriptor keyProviderOp = - new ConstantTupleSourceOperatorDescriptor(spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(), - tb.getSize()); + ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(spec, + keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(), tb.getSize()); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, keyProviderOp, NC1_ID); int[] keyFields = { 0, 1, 2, 3 }; - RTreeSearchOperatorDescriptor secondarySearchOp = - new RTreeSearchOperatorDescriptor(spec, secondaryWithFilterRecDesc, storageManager, lcManagerProvider, - secondarySplitProvider, secondaryTypeTraits, secondaryComparatorFactories, keyFields, - rtreeDataflowHelperFactory, false, false, NoopMissingWriterFactory.INSTANCE, - NoOpOperationCallbackFactory.INSTANCE, true, null, null, pageManagerFactory); + RTreeSearchOperatorDescriptor secondarySearchOp = new RTreeSearchOperatorDescriptor(spec, + secondaryWithFilterRecDesc, keyFields, true, true, secondaryHelperFactory, false, false, null, + NoOpOperationCallbackFactory.INSTANCE, null, null, false); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondarySearchOp, NC1_ID); IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { createFile(nc1) });
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java index 0d4bf92..c44544a 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java +++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java @@ -19,6 +19,9 @@ package org.apache.hyracks.tests.am.rtree; +import java.io.DataOutput; +import java.io.File; + import org.apache.hyracks.api.constraints.PartitionConstraintHelper; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory; @@ -46,26 +49,26 @@ import org.apache.hyracks.dataflow.std.file.IFileSplitProvider; import org.apache.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor; import org.apache.hyracks.dataflow.std.misc.NullSinkOperatorDescriptor; import org.apache.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor; -import org.apache.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory; +import org.apache.hyracks.storage.am.btree.dataflow.BTreeResourceFactory; import org.apache.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor; -import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider; +import org.apache.hyracks.storage.am.common.api.IIndexBuilderFactory; import org.apache.hyracks.storage.am.common.api.IPageManagerFactory; import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory; +import org.apache.hyracks.storage.am.common.build.IndexBuilderFactory; import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; +import org.apache.hyracks.storage.am.common.dataflow.IndexCreateOperatorDescriptor; import org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor; +import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory; import org.apache.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor; -import org.apache.hyracks.storage.am.common.dataflow.TreeIndexCreateOperatorDescriptor; import org.apache.hyracks.storage.am.common.dataflow.TreeIndexInsertUpdateDeleteOperatorDescriptor; import org.apache.hyracks.storage.am.common.freepage.AppendOnlyLinkedMetadataPageManagerFactory; -import org.apache.hyracks.storage.am.common.freepage.LinkedMetadataPageManagerFactory; import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory; import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; import org.apache.hyracks.storage.am.lsm.rtree.utils.LSMRTreeUtils; import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType; import org.apache.hyracks.storage.am.rtree.util.RTreeUtils; +import org.apache.hyracks.storage.common.IResourceFactory; import org.apache.hyracks.storage.common.IStorageManager; -import org.apache.hyracks.storage.common.file.TransientLocalResourceFactoryProvider; -import org.apache.hyracks.test.support.TestIndexLifecycleManagerProvider; import org.apache.hyracks.test.support.TestStorageManager; import org.apache.hyracks.test.support.TestStorageManagerComponentHolder; import org.apache.hyracks.tests.am.common.ITreeIndexOperatorTestHelper; @@ -73,9 +76,6 @@ import org.apache.hyracks.tests.integration.AbstractIntegrationTest; import org.junit.After; import org.junit.Before; -import java.io.DataOutput; -import java.io.File; - public abstract class AbstractRTreeOperatorTest extends AbstractIntegrationTest { static { TestStorageManagerComponentHolder.init(8192, 20, 20); @@ -90,16 +90,16 @@ public abstract class AbstractRTreeOperatorTest extends AbstractIntegrationTest protected RTreeType rTreeType; protected final IStorageManager storageManager = new TestStorageManager(); - protected final IIndexLifecycleManagerProvider lcManagerProvider = new TestIndexLifecycleManagerProvider(); protected final IPageManagerFactory pageManagerFactory = AppendOnlyLinkedMetadataPageManagerFactory.INSTANCE; - protected IIndexDataflowHelperFactory rtreeDataflowHelperFactory; - protected IIndexDataflowHelperFactory btreeDataflowHelperFactory = new BTreeDataflowHelperFactory(true); + protected IResourceFactory rtreeFactory; + protected IResourceFactory btreeFactory; // field, type and key declarations for primary index protected final int primaryFieldCount = 10; protected final ITypeTraits[] primaryTypeTraits = new ITypeTraits[primaryFieldCount]; protected final int primaryKeyFieldCount = 1; - protected final IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount]; + protected final IBinaryComparatorFactory[] primaryComparatorFactories = + new IBinaryComparatorFactory[primaryKeyFieldCount]; protected final RecordDescriptor primaryRecDesc = new RecordDescriptor(new ISerializerDeserializer[] { new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), @@ -111,23 +111,25 @@ public abstract class AbstractRTreeOperatorTest extends AbstractIntegrationTest // to be set by subclasses protected String primaryFileName; protected IFileSplitProvider primarySplitProvider; + protected IIndexDataflowHelperFactory primaryHelperFactory; // field, type and key declarations for secondary indexes protected final int secondaryFieldCount = 5; protected final ITypeTraits[] secondaryTypeTraits = new ITypeTraits[secondaryFieldCount]; protected final int secondaryKeyFieldCount = 4; - protected final IBinaryComparatorFactory[] secondaryComparatorFactories = new IBinaryComparatorFactory[secondaryKeyFieldCount]; + protected final IBinaryComparatorFactory[] secondaryComparatorFactories = + new IBinaryComparatorFactory[secondaryKeyFieldCount]; - protected final RecordDescriptor secondaryRecDesc = new RecordDescriptor(new ISerializerDeserializer[] { - DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, - DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, - new UTF8StringSerializerDeserializer() }); + protected final RecordDescriptor secondaryRecDesc = + new RecordDescriptor(new ISerializerDeserializer[] { DoubleSerializerDeserializer.INSTANCE, + DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, + DoubleSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer() }); - protected final RecordDescriptor secondaryWithFilterRecDesc = new RecordDescriptor(new ISerializerDeserializer[] { - DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, - DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, - new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), - new UTF8StringSerializerDeserializer() }); + protected final RecordDescriptor secondaryWithFilterRecDesc = + new RecordDescriptor(new ISerializerDeserializer[] { DoubleSerializerDeserializer.INSTANCE, + DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, + DoubleSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer(), + new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() }); // This is only used for the LSMRTree. We need a comparator Factories for // the BTree component of the LSMRTree. @@ -136,6 +138,7 @@ public abstract class AbstractRTreeOperatorTest extends AbstractIntegrationTest protected String secondaryFileName; protected IFileSplitProvider secondarySplitProvider; + protected IIndexDataflowHelperFactory secondaryHelperFactory; protected ITreeIndexOperatorTestHelper testHelper; @@ -150,9 +153,11 @@ public abstract class AbstractRTreeOperatorTest extends AbstractIntegrationTest primaryFileName = testHelper.getPrimaryIndexName(); primarySplitProvider = new ConstantFileSplitProvider(new FileSplit[] { new ManagedFileSplit(NC1_ID, primaryFileName) }); + primaryHelperFactory = new IndexDataflowHelperFactory(storageManager, primarySplitProvider); secondaryFileName = testHelper.getSecondaryIndexName(); secondarySplitProvider = new ConstantFileSplitProvider(new FileSplit[] { new ManagedFileSplit(NC1_ID, secondaryFileName) }); + secondaryHelperFactory = new IndexDataflowHelperFactory(storageManager, secondarySplitProvider); // field, type and key declarations for primary index primaryTypeTraits[0] = UTF8StringPointable.TYPE_TRAITS; @@ -208,27 +213,25 @@ public abstract class AbstractRTreeOperatorTest extends AbstractIntegrationTest IPrimitiveValueProviderFactory[] secondaryValueProviderFactories = RTreeUtils .createPrimitiveValueProviderFactories(secondaryComparatorFactories.length, DoublePointable.FACTORY); - rtreeDataflowHelperFactory = - createDataFlowHelperFactory(secondaryValueProviderFactories, RTreePolicyType.RSTARTREE, - btreeComparatorFactories, - LSMRTreeUtils.proposeBestLinearizer(secondaryTypeTraits, secondaryComparatorFactories.length), - btreeFields, rtreeFields, filterTypes, filterCmpFactories, filterFields); + rtreeFactory = createSecondaryResourceFactory(secondaryValueProviderFactories, RTreePolicyType.RSTARTREE, + btreeComparatorFactories, + LSMRTreeUtils.proposeBestLinearizer(secondaryTypeTraits, secondaryComparatorFactories.length), + btreeFields); } - protected abstract IIndexDataflowHelperFactory createDataFlowHelperFactory( + protected abstract IResourceFactory createSecondaryResourceFactory( IPrimitiveValueProviderFactory[] secondaryValueProviderFactories, RTreePolicyType rtreePolicyType, IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory, - int[] btreeFields, int[] rtree, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, - int[] filterFields) throws HyracksDataException; + int[] btreeFields); protected void createPrimaryIndex() throws Exception { JobSpecification spec = new JobSpecification(); - TransientLocalResourceFactoryProvider localResourceFactoryProvider = new TransientLocalResourceFactoryProvider(); - TreeIndexCreateOperatorDescriptor primaryCreateOp = new TreeIndexCreateOperatorDescriptor(spec, storageManager, - lcManagerProvider, primarySplitProvider, primaryTypeTraits, primaryComparatorFactories, null, - btreeDataflowHelperFactory, localResourceFactoryProvider, NoOpOperationCallbackFactory.INSTANCE, + btreeFactory = new BTreeResourceFactory(storageManager, primaryTypeTraits, primaryComparatorFactories, pageManagerFactory); + IIndexBuilderFactory indexBuilderFactory = + new IndexBuilderFactory(storageManager, primarySplitProvider, btreeFactory, false); + IndexCreateOperatorDescriptor primaryCreateOp = new IndexCreateOperatorDescriptor(spec, indexBuilderFactory); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryCreateOp, NC1_ID); spec.addRoot(primaryCreateOp); runTest(spec); @@ -237,17 +240,17 @@ public abstract class AbstractRTreeOperatorTest extends AbstractIntegrationTest protected void loadPrimaryIndex() throws Exception { JobSpecification spec = new JobSpecification(); - FileSplit[] ordersSplits = new FileSplit[] { new ManagedFileSplit(NC1_ID, - "data" + File.separator + "orders-with-locations-part1.txt") }; + FileSplit[] ordersSplits = new FileSplit[] { + new ManagedFileSplit(NC1_ID, "data" + File.separator + "orders-with-locations-part1.txt") }; IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(ordersSplits); - RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] { - new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), - new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), - new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), - new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), - new UTF8StringSerializerDeserializer(), DoubleSerializerDeserializer.INSTANCE, - DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, - DoubleSerializerDeserializer.INSTANCE }); + RecordDescriptor ordersDesc = + new RecordDescriptor(new ISerializerDeserializer[] { new UTF8StringSerializerDeserializer(), + new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), + new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), + new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), + new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), + DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, + DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE }); FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(spec, ordersSplitProvider, new DelimitedDataTupleParserFactory(new IValueParserFactory[] { UTF8StringParserFactory.INSTANCE, @@ -256,7 +259,8 @@ public abstract class AbstractRTreeOperatorTest extends AbstractIntegrationTest UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE, DoubleParserFactory.INSTANCE, DoubleParserFactory.INSTANCE, DoubleParserFactory.INSTANCE, - DoubleParserFactory.INSTANCE }, '|'), ordersDesc); + DoubleParserFactory.INSTANCE }, '|'), + ordersDesc); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, ordScanner, NC1_ID); ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(spec, 1000, new int[] { 0 }, @@ -266,9 +270,7 @@ public abstract class AbstractRTreeOperatorTest extends AbstractIntegrationTest int[] fieldPermutation = { 0, 1, 2, 4, 5, 7, 9, 10, 11, 12 }; TreeIndexBulkLoadOperatorDescriptor primaryBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec, - primaryRecDesc, storageManager, lcManagerProvider, primarySplitProvider, primaryTypeTraits, - primaryComparatorFactories, null, fieldPermutation, 0.7f, false, 1000L, true, - btreeDataflowHelperFactory, pageManagerFactory); + primaryRecDesc, fieldPermutation, 0.7f, false, 1000L, true, primaryHelperFactory); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBulkLoad, NC1_ID); NullSinkOperatorDescriptor nsOpDesc = new NullSinkOperatorDescriptor(spec); @@ -286,11 +288,9 @@ public abstract class AbstractRTreeOperatorTest extends AbstractIntegrationTest protected void createSecondaryIndex() throws Exception { JobSpecification spec = new JobSpecification(); - TransientLocalResourceFactoryProvider localResourceFactoryProvider = new TransientLocalResourceFactoryProvider(); - TreeIndexCreateOperatorDescriptor secondaryCreateOp = new TreeIndexCreateOperatorDescriptor(spec, - storageManager, lcManagerProvider, secondarySplitProvider, secondaryTypeTraits, - secondaryComparatorFactories, null, rtreeDataflowHelperFactory, localResourceFactoryProvider, - NoOpOperationCallbackFactory.INSTANCE, pageManagerFactory); + IndexBuilderFactory indexBuilderFactory = + new IndexBuilderFactory(storageManager, secondarySplitProvider, rtreeFactory, false); + IndexCreateOperatorDescriptor secondaryCreateOp = new IndexCreateOperatorDescriptor(spec, indexBuilderFactory); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryCreateOp, NC1_ID); spec.addRoot(secondaryCreateOp); runTest(spec); @@ -307,8 +307,8 @@ public abstract class AbstractRTreeOperatorTest extends AbstractIntegrationTest new UTF8StringSerializerDeserializer().serialize("0", dos); tb.addFieldEndOffset(); - ISerializerDeserializer[] keyRecDescSers = { new UTF8StringSerializerDeserializer(), - new UTF8StringSerializerDeserializer() }; + ISerializerDeserializer[] keyRecDescSers = + { new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() }; RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers); ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(spec, @@ -320,17 +320,14 @@ public abstract class AbstractRTreeOperatorTest extends AbstractIntegrationTest // scan primary index BTreeSearchOperatorDescriptor primarySearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc, - storageManager, lcManagerProvider, primarySplitProvider, primaryTypeTraits, primaryComparatorFactories, - null, lowKeyFields, highKeyFields, true, true, btreeDataflowHelperFactory, false, false, null, - NoOpOperationCallbackFactory.INSTANCE, null, null, new LinkedMetadataPageManagerFactory()); + lowKeyFields, highKeyFields, true, true, primaryHelperFactory, false, false, null, + NoOpOperationCallbackFactory.INSTANCE, null, null, false); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primarySearchOp, NC1_ID); // load secondary index int[] fieldPermutation = { 6, 7, 8, 9, 0 }; TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec, - secondaryRecDesc, storageManager, lcManagerProvider, secondarySplitProvider, secondaryTypeTraits, - secondaryComparatorFactories, null, fieldPermutation, 0.7f, false, 1000L, true, - rtreeDataflowHelperFactory, pageManagerFactory); + secondaryRecDesc, fieldPermutation, 0.7f, false, 1000L, true, secondaryHelperFactory); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryBulkLoad, NC1_ID); NullSinkOperatorDescriptor nsOpDesc = new NullSinkOperatorDescriptor(spec); @@ -348,17 +345,17 @@ public abstract class AbstractRTreeOperatorTest extends AbstractIntegrationTest JobSpecification spec = new JobSpecification(); - FileSplit[] ordersSplits = new FileSplit[] { new ManagedFileSplit(NC1_ID, - "data" + File.separator + "orders-with-locations-part2.txt") }; + FileSplit[] ordersSplits = new FileSplit[] { + new ManagedFileSplit(NC1_ID, "data" + File.separator + "orders-with-locations-part2.txt") }; IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(ordersSplits); - RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] { - new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), - new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), - new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), - new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), - new UTF8StringSerializerDeserializer(), DoubleSerializerDeserializer.INSTANCE, - DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, - DoubleSerializerDeserializer.INSTANCE }); + RecordDescriptor ordersDesc = + new RecordDescriptor(new ISerializerDeserializer[] { new UTF8StringSerializerDeserializer(), + new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), + new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), + new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), + new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), + DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, + DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE }); FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(spec, ordersSplitProvider, new DelimitedDataTupleParserFactory(new IValueParserFactory[] { UTF8StringParserFactory.INSTANCE, @@ -367,23 +364,22 @@ public abstract class AbstractRTreeOperatorTest extends AbstractIntegrationTest UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE, DoubleParserFactory.INSTANCE, DoubleParserFactory.INSTANCE, DoubleParserFactory.INSTANCE, - DoubleParserFactory.INSTANCE }, '|'), ordersDesc); + DoubleParserFactory.INSTANCE }, '|'), + ordersDesc); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, ordScanner, NC1_ID); // insert into primary index int[] primaryFieldPermutation = { 0, 1, 2, 4, 5, 7, 9, 10, 11, 12 }; - TreeIndexInsertUpdateDeleteOperatorDescriptor primaryInsertOp = new TreeIndexInsertUpdateDeleteOperatorDescriptor( - spec, ordersDesc, storageManager, lcManagerProvider, primarySplitProvider, primaryTypeTraits, - primaryComparatorFactories, null, primaryFieldPermutation, IndexOperation.INSERT, - btreeDataflowHelperFactory, null, NoOpOperationCallbackFactory.INSTANCE, pageManagerFactory); + TreeIndexInsertUpdateDeleteOperatorDescriptor primaryInsertOp = + new TreeIndexInsertUpdateDeleteOperatorDescriptor(spec, ordersDesc, primaryFieldPermutation, + IndexOperation.INSERT, primaryHelperFactory, null, NoOpOperationCallbackFactory.INSTANCE); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryInsertOp, NC1_ID); // secondary index int[] secondaryFieldPermutation = { 9, 10, 11, 12, 0 }; - TreeIndexInsertUpdateDeleteOperatorDescriptor secondaryInsertOp = new TreeIndexInsertUpdateDeleteOperatorDescriptor( - spec, ordersDesc, storageManager, lcManagerProvider, secondarySplitProvider, secondaryTypeTraits, - secondaryComparatorFactories, null, secondaryFieldPermutation, IndexOperation.INSERT, - rtreeDataflowHelperFactory, null, NoOpOperationCallbackFactory.INSTANCE, pageManagerFactory); + TreeIndexInsertUpdateDeleteOperatorDescriptor secondaryInsertOp = + new TreeIndexInsertUpdateDeleteOperatorDescriptor(spec, ordersDesc, secondaryFieldPermutation, + IndexOperation.INSERT, secondaryHelperFactory, null, NoOpOperationCallbackFactory.INSTANCE); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryInsertOp, NC1_ID); NullSinkOperatorDescriptor nullSink = new NullSinkOperatorDescriptor(spec); @@ -401,8 +397,7 @@ public abstract class AbstractRTreeOperatorTest extends AbstractIntegrationTest protected void destroyPrimaryIndex() throws Exception { JobSpecification spec = new JobSpecification(); - IndexDropOperatorDescriptor primaryDropOp = new IndexDropOperatorDescriptor(spec, storageManager, - lcManagerProvider, primarySplitProvider, btreeDataflowHelperFactory, pageManagerFactory); + IndexDropOperatorDescriptor primaryDropOp = new IndexDropOperatorDescriptor(spec, primaryHelperFactory); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryDropOp, NC1_ID); spec.addRoot(primaryDropOp); runTest(spec); @@ -410,8 +405,7 @@ public abstract class AbstractRTreeOperatorTest extends AbstractIntegrationTest protected void destroySecondaryIndex() throws Exception { JobSpecification spec = new JobSpecification(); - IndexDropOperatorDescriptor secondaryDropOp = new IndexDropOperatorDescriptor(spec, storageManager, - lcManagerProvider, secondarySplitProvider, rtreeDataflowHelperFactory, pageManagerFactory); + IndexDropOperatorDescriptor secondaryDropOp = new IndexDropOperatorDescriptor(spec, secondaryHelperFactory); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryDropOp, NC1_ID); spec.addRoot(secondaryDropOp); runTest(spec); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeOperatorTestHelper.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeOperatorTestHelper.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeOperatorTestHelper.java index 0940819..43d685e 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeOperatorTestHelper.java +++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeOperatorTestHelper.java @@ -20,18 +20,24 @@ package org.apache.hyracks.tests.am.rtree; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; +import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory; +import org.apache.hyracks.api.dataflow.value.ITypeTraits; +import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory; import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory; -import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; -import org.apache.hyracks.storage.am.rtree.dataflow.RTreeDataflowHelperFactory; +import org.apache.hyracks.storage.am.rtree.dataflow.RTreeResourceFactory; import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType; +import org.apache.hyracks.storage.common.IResourceFactory; +import org.apache.hyracks.storage.common.IStorageManager; import org.apache.hyracks.tests.am.common.TreeOperatorTestHelper; public class RTreeOperatorTestHelper extends TreeOperatorTestHelper { - public IIndexDataflowHelperFactory createDataFlowHelperFactory( + public IResourceFactory getSecondaryLocalResourceFactory(IStorageManager storageManager, IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType, - IBinaryComparatorFactory[] btreeComparatorFactories, boolean durable) { - return new RTreeDataflowHelperFactory(valueProviderFactories, rtreePolicyType, durable); + IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory, + int[] btreeFields, ITypeTraits[] secondaryTypeTraits, + IBinaryComparatorFactory[] secondaryComparatorFactories, IMetadataPageManagerFactory pageManagerFactory) { + return new RTreeResourceFactory(storageManager, secondaryTypeTraits, secondaryComparatorFactories, + pageManagerFactory, valueProviderFactories, rtreePolicyType); } - } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexInsertOperatorTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexInsertOperatorTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexInsertOperatorTest.java index 3fe64c7..784cd10 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexInsertOperatorTest.java +++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexInsertOperatorTest.java @@ -23,8 +23,10 @@ import java.io.DataOutput; import org.apache.hyracks.api.constraints.PartitionConstraintHelper; import org.apache.hyracks.api.dataflow.IOperatorDescriptor; -import org.apache.hyracks.api.dataflow.value.*; -import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; +import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory; +import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; +import org.apache.hyracks.api.dataflow.value.RecordDescriptor; import org.apache.hyracks.api.io.FileSplit; import org.apache.hyracks.api.job.JobSpecification; import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder; @@ -35,12 +37,12 @@ import org.apache.hyracks.dataflow.std.file.IFileSplitProvider; import org.apache.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor; import org.apache.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor; import org.apache.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor; +import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory; import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory; -import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; -import org.apache.hyracks.storage.am.common.freepage.LinkedMetadataPageManagerFactory; import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory; import org.apache.hyracks.storage.am.rtree.dataflow.RTreeSearchOperatorDescriptor; import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType; +import org.apache.hyracks.storage.common.IResourceFactory; import org.junit.Before; import org.junit.Test; @@ -76,18 +78,17 @@ public class RTreeSecondaryIndexInsertOperatorTest extends AbstractRTreeOperator tb.addFieldEndOffset(); DoubleSerializerDeserializer.INSTANCE.serialize(-149.024, dos); tb.addFieldEndOffset(); - ISerializerDeserializer[] keyRecDescSers = { DoubleSerializerDeserializer.INSTANCE, - DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, - DoubleSerializerDeserializer.INSTANCE }; + ISerializerDeserializer[] keyRecDescSers = + { DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, + DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE }; RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers); ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(), tb.getSize()); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, keyProviderOp, NC1_ID); int[] keyFields = { 0, 1, 2, 3 }; - RTreeSearchOperatorDescriptor secondarySearchOp = new RTreeSearchOperatorDescriptor(spec, secondaryRecDesc, - storageManager, lcManagerProvider, secondarySplitProvider, secondaryTypeTraits, - secondaryComparatorFactories, keyFields, rtreeDataflowHelperFactory, false, false, null, - NoOpOperationCallbackFactory.INSTANCE, null, null, pageManagerFactory); + RTreeSearchOperatorDescriptor secondarySearchOp = + new RTreeSearchOperatorDescriptor(spec, secondaryRecDesc, keyFields, true, true, secondaryHelperFactory, + false, false, null, NoOpOperationCallbackFactory.INSTANCE, null, null, false); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondarySearchOp, NC1_ID); // fifth field from the tuples coming from secondary index int[] primaryLowKeyFields = { 4 }; @@ -95,9 +96,8 @@ public class RTreeSecondaryIndexInsertOperatorTest extends AbstractRTreeOperator int[] primaryHighKeyFields = { 4 }; // search primary index BTreeSearchOperatorDescriptor primarySearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc, - storageManager, lcManagerProvider, primarySplitProvider, primaryTypeTraits, primaryComparatorFactories, - null, primaryLowKeyFields, primaryHighKeyFields, true, true, btreeDataflowHelperFactory, false, false, - null, NoOpOperationCallbackFactory.INSTANCE, null, null, new LinkedMetadataPageManagerFactory()); + primaryLowKeyFields, primaryHighKeyFields, true, true, primaryHelperFactory, false, false, null, + NoOpOperationCallbackFactory.INSTANCE, null, null, false); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primarySearchOp, NC1_ID); IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { createFile(nc1) }); IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ","); @@ -110,13 +110,14 @@ public class RTreeSecondaryIndexInsertOperatorTest extends AbstractRTreeOperator } @Override - protected IIndexDataflowHelperFactory createDataFlowHelperFactory( + protected IResourceFactory createSecondaryResourceFactory( IPrimitiveValueProviderFactory[] secondaryValueProviderFactories, RTreePolicyType rtreePolicyType, IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory, - int[] btreeFields, int [] rtreeFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, - int[] filterFields) throws HyracksDataException { - return ((RTreeOperatorTestHelper) testHelper).createDataFlowHelperFactory(secondaryValueProviderFactories, - rtreePolicyType, null, true); + int[] btreeFields) { + return ((RTreeOperatorTestHelper) testHelper).getSecondaryLocalResourceFactory(storageManager, + secondaryValueProviderFactories, rtreePolicyType, btreeComparatorFactories, linearizerCmpFactory, + btreeFields, secondaryTypeTraits, secondaryComparatorFactories, + (IMetadataPageManagerFactory) pageManagerFactory); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexScanOperatorTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexScanOperatorTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexScanOperatorTest.java index 0cf9c52..ff44d10 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexScanOperatorTest.java +++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexScanOperatorTest.java @@ -23,7 +23,10 @@ import java.io.DataOutput; import org.apache.hyracks.api.constraints.PartitionConstraintHelper; import org.apache.hyracks.api.dataflow.IOperatorDescriptor; -import org.apache.hyracks.api.dataflow.value.*; +import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; +import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory; +import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; +import org.apache.hyracks.api.dataflow.value.RecordDescriptor; import org.apache.hyracks.api.io.FileSplit; import org.apache.hyracks.api.job.JobSpecification; import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder; @@ -33,11 +36,12 @@ import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider; import org.apache.hyracks.dataflow.std.file.IFileSplitProvider; import org.apache.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor; import org.apache.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor; +import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory; import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory; -import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory; import org.apache.hyracks.storage.am.rtree.dataflow.RTreeSearchOperatorDescriptor; import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType; +import org.apache.hyracks.storage.common.IResourceFactory; import org.junit.Before; import org.junit.Test; @@ -72,18 +76,17 @@ public class RTreeSecondaryIndexScanOperatorTest extends AbstractRTreeOperatorTe tb.addFieldEndOffset(); DoubleSerializerDeserializer.INSTANCE.serialize(0.0, dos); tb.addFieldEndOffset(); - ISerializerDeserializer[] keyRecDescSers = { DoubleSerializerDeserializer.INSTANCE, - DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, - DoubleSerializerDeserializer.INSTANCE }; + ISerializerDeserializer[] keyRecDescSers = + { DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, + DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE }; RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers); ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(), tb.getSize()); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, keyProviderOp, NC1_ID); int[] keyFields = null; - RTreeSearchOperatorDescriptor secondarySearchOp = new RTreeSearchOperatorDescriptor(spec, secondaryRecDesc, - storageManager, lcManagerProvider, secondarySplitProvider, secondaryTypeTraits, - secondaryComparatorFactories, keyFields, rtreeDataflowHelperFactory, false, false, null, - NoOpOperationCallbackFactory.INSTANCE, null, null, pageManagerFactory); + RTreeSearchOperatorDescriptor secondarySearchOp = + new RTreeSearchOperatorDescriptor(spec, secondaryRecDesc, keyFields, true, true, secondaryHelperFactory, + false, false, null, NoOpOperationCallbackFactory.INSTANCE, null, null, false); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondarySearchOp, NC1_ID); IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { createFile(nc1) }); IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ","); @@ -95,18 +98,19 @@ public class RTreeSecondaryIndexScanOperatorTest extends AbstractRTreeOperatorTe } @Override - protected IIndexDataflowHelperFactory createDataFlowHelperFactory( - IPrimitiveValueProviderFactory[] secondaryValueProviderFactories, RTreePolicyType rtreePolicyType, - IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory, - int[] btreeFields, int[] rtreeFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, - int[] filterFields) { - return ((RTreeOperatorTestHelper) testHelper).createDataFlowHelperFactory(secondaryValueProviderFactories, - rtreePolicyType, null, true); - } - - @Override public void cleanup() throws Exception { destroyPrimaryIndex(); destroySecondaryIndex(); } + + @Override + protected IResourceFactory createSecondaryResourceFactory( + IPrimitiveValueProviderFactory[] secondaryValueProviderFactories, RTreePolicyType rtreePolicyType, + IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory, + int[] btreeFields) { + return ((RTreeOperatorTestHelper) testHelper).getSecondaryLocalResourceFactory(storageManager, + secondaryValueProviderFactories, rtreePolicyType, btreeComparatorFactories, linearizerCmpFactory, + btreeFields, secondaryTypeTraits, secondaryComparatorFactories, + (IMetadataPageManagerFactory) pageManagerFactory); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexSearchOperatorTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexSearchOperatorTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexSearchOperatorTest.java index 8818282..e234322 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexSearchOperatorTest.java +++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexSearchOperatorTest.java @@ -23,8 +23,10 @@ import java.io.DataOutput; import org.apache.hyracks.api.constraints.PartitionConstraintHelper; import org.apache.hyracks.api.dataflow.IOperatorDescriptor; -import org.apache.hyracks.api.dataflow.value.*; -import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; +import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory; +import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; +import org.apache.hyracks.api.dataflow.value.RecordDescriptor; import org.apache.hyracks.api.io.FileSplit; import org.apache.hyracks.api.job.JobSpecification; import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder; @@ -35,12 +37,12 @@ import org.apache.hyracks.dataflow.std.file.IFileSplitProvider; import org.apache.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor; import org.apache.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor; import org.apache.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor; +import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory; import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory; -import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; -import org.apache.hyracks.storage.am.common.freepage.LinkedMetadataPageManagerFactory; import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory; import org.apache.hyracks.storage.am.rtree.dataflow.RTreeSearchOperatorDescriptor; import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType; +import org.apache.hyracks.storage.common.IResourceFactory; import org.junit.Before; import org.junit.Test; @@ -74,18 +76,17 @@ public class RTreeSecondaryIndexSearchOperatorTest extends AbstractRTreeOperator tb.addFieldEndOffset(); DoubleSerializerDeserializer.INSTANCE.serialize(-149.024, dos); tb.addFieldEndOffset(); - ISerializerDeserializer[] keyRecDescSers = { DoubleSerializerDeserializer.INSTANCE, - DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, - DoubleSerializerDeserializer.INSTANCE }; + ISerializerDeserializer[] keyRecDescSers = + { DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, + DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE }; RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers); ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(), tb.getSize()); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, keyProviderOp, NC1_ID); int[] keyFields = { 0, 1, 2, 3 }; - RTreeSearchOperatorDescriptor secondarySearchOp = new RTreeSearchOperatorDescriptor(spec, secondaryRecDesc, - storageManager, lcManagerProvider, secondarySplitProvider, secondaryTypeTraits, - secondaryComparatorFactories, keyFields, rtreeDataflowHelperFactory, false, false, null, - NoOpOperationCallbackFactory.INSTANCE, null, null, pageManagerFactory); + RTreeSearchOperatorDescriptor secondarySearchOp = + new RTreeSearchOperatorDescriptor(spec, secondaryRecDesc, keyFields, true, true, secondaryHelperFactory, + false, false, null, NoOpOperationCallbackFactory.INSTANCE, null, null, false); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondarySearchOp, NC1_ID); // fifth field from the tuples coming from secondary index int[] primaryLowKeyFields = { 4 }; @@ -93,9 +94,8 @@ public class RTreeSecondaryIndexSearchOperatorTest extends AbstractRTreeOperator int[] primaryHighKeyFields = { 4 }; // search primary index BTreeSearchOperatorDescriptor primarySearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc, - storageManager, lcManagerProvider, primarySplitProvider, primaryTypeTraits, primaryComparatorFactories, - null, primaryLowKeyFields, primaryHighKeyFields, true, true, btreeDataflowHelperFactory, false, false, - null, NoOpOperationCallbackFactory.INSTANCE, null, null, new LinkedMetadataPageManagerFactory()); + primaryLowKeyFields, primaryHighKeyFields, true, true, primaryHelperFactory, false, false, null, + NoOpOperationCallbackFactory.INSTANCE, null, null, false); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primarySearchOp, NC1_ID); IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { createFile(nc1) }); IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ","); @@ -108,13 +108,14 @@ public class RTreeSecondaryIndexSearchOperatorTest extends AbstractRTreeOperator } @Override - protected IIndexDataflowHelperFactory createDataFlowHelperFactory( + protected IResourceFactory createSecondaryResourceFactory( IPrimitiveValueProviderFactory[] secondaryValueProviderFactories, RTreePolicyType rtreePolicyType, IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory, - int[] btreeFields, int [] rtreeFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, - int[] filterFields) throws HyracksDataException { - return ((RTreeOperatorTestHelper) testHelper).createDataFlowHelperFactory(secondaryValueProviderFactories, - rtreePolicyType, null, true); + int[] btreeFields) { + return ((RTreeOperatorTestHelper) testHelper).getSecondaryLocalResourceFactory(storageManager, + secondaryValueProviderFactories, rtreePolicyType, btreeComparatorFactories, linearizerCmpFactory, + btreeFields, secondaryTypeTraits, secondaryComparatorFactories, + (IMetadataPageManagerFactory) pageManagerFactory); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexStatsOperatorTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexStatsOperatorTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexStatsOperatorTest.java index 59a57f6..81c47e6 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexStatsOperatorTest.java +++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/rtree/RTreeSecondaryIndexStatsOperatorTest.java @@ -23,18 +23,17 @@ import org.apache.hyracks.api.constraints.PartitionConstraintHelper; import org.apache.hyracks.api.dataflow.IOperatorDescriptor; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory; -import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.api.io.FileSplit; import org.apache.hyracks.api.job.JobSpecification; import org.apache.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor; import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider; import org.apache.hyracks.dataflow.std.file.IFileSplitProvider; import org.apache.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor; +import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory; import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory; -import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; import org.apache.hyracks.storage.am.common.dataflow.TreeIndexStatsOperatorDescriptor; -import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory; import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType; +import org.apache.hyracks.storage.common.IResourceFactory; import org.junit.Before; import org.junit.Test; @@ -56,9 +55,8 @@ public class RTreeSecondaryIndexStatsOperatorTest extends AbstractRTreeOperatorT @Test public void showPrimaryIndexStats() throws Exception { JobSpecification spec = new JobSpecification(); - TreeIndexStatsOperatorDescriptor secondaryStatsOp = new TreeIndexStatsOperatorDescriptor(spec, storageManager, - lcManagerProvider, secondarySplitProvider, secondaryTypeTraits, secondaryComparatorFactories, null, - rtreeDataflowHelperFactory, NoOpOperationCallbackFactory.INSTANCE, pageManagerFactory); + TreeIndexStatsOperatorDescriptor secondaryStatsOp = + new TreeIndexStatsOperatorDescriptor(spec, storageManager, secondaryHelperFactory); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryStatsOp, NC1_ID); IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { createFile(nc1) }); IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ","); @@ -69,18 +67,19 @@ public class RTreeSecondaryIndexStatsOperatorTest extends AbstractRTreeOperatorT } @Override - protected IIndexDataflowHelperFactory createDataFlowHelperFactory( - IPrimitiveValueProviderFactory[] secondaryValueProviderFactories, RTreePolicyType rtreePolicyType, - IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory, - int[] btreeFields, int[] rtreeFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, - int[] filterFields) { - return ((RTreeOperatorTestHelper) testHelper).createDataFlowHelperFactory(secondaryValueProviderFactories, - rtreePolicyType, null, true); - } - - @Override public void cleanup() throws Exception { destroyPrimaryIndex(); destroySecondaryIndex(); } + + @Override + protected IResourceFactory createSecondaryResourceFactory( + IPrimitiveValueProviderFactory[] secondaryValueProviderFactories, RTreePolicyType rtreePolicyType, + IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory, + int[] btreeFields) { + return ((RTreeOperatorTestHelper) testHelper).getSecondaryLocalResourceFactory(storageManager, + secondaryValueProviderFactories, rtreePolicyType, btreeComparatorFactories, linearizerCmpFactory, + btreeFields, secondaryTypeTraits, secondaryComparatorFactories, + (IMetadataPageManagerFactory) pageManagerFactory); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/JobFailureTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/JobFailureTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/JobFailureTest.java index 6571f7e..871109a 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/JobFailureTest.java +++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/JobFailureTest.java @@ -32,7 +32,7 @@ public class JobFailureTest extends AbstractMultiNCIntegrationTest { @Test public void failureOnCreatePushRuntime() throws Exception { - for (int round = 0; round < 1000; ++round) { + for (int round = 0; round < 100; ++round) { execTest(); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/HDFSReadOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/HDFSReadOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/HDFSReadOperatorDescriptor.java index 78c7c6a..8357ae0 100644 --- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/HDFSReadOperatorDescriptor.java +++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/HDFSReadOperatorDescriptor.java @@ -84,7 +84,7 @@ public class HDFSReadOperatorDescriptor extends AbstractSingleActivityOperatorDe this.executed = new boolean[scheduledLocations.length]; Arrays.fill(executed, false); this.tupleParserFactory = tupleParserFactory; - this.recordDescriptors[0] = rd; + this.outRecDescs[0] = rd; } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs2/dataflow/HDFSReadOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs2/dataflow/HDFSReadOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs2/dataflow/HDFSReadOperatorDescriptor.java index cd55098..15bf260 100644 --- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs2/dataflow/HDFSReadOperatorDescriptor.java +++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs2/dataflow/HDFSReadOperatorDescriptor.java @@ -97,7 +97,7 @@ public class HDFSReadOperatorDescriptor extends AbstractSingleActivityOperatorDe this.executed = new boolean[scheduledLocations.length]; Arrays.fill(executed, false); this.tupleParserFactory = tupleParserFactory; - this.recordDescriptors[0] = rd; + this.outRecDescs[0] = rd; } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/pom.xml index 659544e..18343f0 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/pom.xml @@ -43,11 +43,6 @@ <dependencies> <dependency> <groupId>org.apache.hyracks</groupId> - <artifactId>hyracks-storage-am-common</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.hyracks</groupId> <artifactId>hyracks-dataflow-common</artifactId> <version>${project.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java b/hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java index dee8271..3b49d08 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java @@ -24,7 +24,7 @@ import java.nio.ByteBuffer; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; -import org.apache.hyracks.storage.am.common.api.IIndexBulkLoader; +import org.apache.hyracks.storage.common.IIndexBulkLoader; import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICachedPage; import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/hyracks-fullstack/hyracks/hyracks-storage-am-btree/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/pom.xml index 639053f..ec04f27 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/pom.xml @@ -16,7 +16,8 @@ ! specific language governing permissions and limitations ! under the License. !--> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> <modelVersion>4.0.0</modelVersion> <artifactId>hyracks-storage-am-btree</artifactId> <name>hyracks-storage-am-btree</name> @@ -103,5 +104,9 @@ <artifactId>hyracks-data-std</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> + </dependency> </dependencies> -</project> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java index 37e0ab8..ea1bd5f 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java @@ -24,7 +24,7 @@ import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference; import org.apache.hyracks.storage.am.common.ophelpers.FindTupleMode; import org.apache.hyracks.storage.am.common.ophelpers.FindTupleNoExactMatchPolicy; -import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator; +import org.apache.hyracks.storage.common.MultiComparator; import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IPrefixSlotManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IPrefixSlotManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IPrefixSlotManager.java index 44dc69f..ceae674 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IPrefixSlotManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IPrefixSlotManager.java @@ -25,7 +25,7 @@ import org.apache.hyracks.storage.am.common.api.ISlotManager; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference; import org.apache.hyracks.storage.am.common.ophelpers.FindTupleMode; import org.apache.hyracks.storage.am.common.ophelpers.FindTupleNoExactMatchPolicy; -import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator; +import org.apache.hyracks.storage.common.MultiComparator; /** * A slot consists of two fields. The first field is 1 byte and it indicates the slot number of http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java index 00121ec..ac14b94 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java @@ -33,8 +33,8 @@ import org.apache.hyracks.storage.am.btree.impls.FieldPrefixSlotManager; import org.apache.hyracks.storage.am.btree.impls.FieldPrefixTupleReference; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameCompressor; -import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator; import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriter; +import org.apache.hyracks.storage.common.MultiComparator; public class FieldPrefixCompressor implements ITreeIndexFrameCompressor { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java deleted file mode 100644 index 6f6722a..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hyracks.storage.am.btree.dataflow; - -import org.apache.hyracks.api.context.IHyracksTaskContext; -import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.api.io.FileReference; -import org.apache.hyracks.storage.am.btree.frames.BTreeLeafFrameType; -import org.apache.hyracks.storage.am.btree.util.BTreeUtils; -import org.apache.hyracks.storage.am.common.api.ITreeIndex; -import org.apache.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor; -import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor; -import org.apache.hyracks.storage.am.common.dataflow.TreeIndexDataflowHelper; -import org.apache.hyracks.storage.am.common.util.IndexFileNameUtil; -import org.apache.hyracks.storage.common.buffercache.IBufferCache; - -public class BTreeDataflowHelper extends TreeIndexDataflowHelper { - - public BTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition, - boolean durable) throws HyracksDataException { - super(opDesc, ctx, partition, durable); - } - - @Override - public ITreeIndex createIndexInstance() throws HyracksDataException { - AbstractTreeIndexOperatorDescriptor treeOpDesc = (AbstractTreeIndexOperatorDescriptor) opDesc; - FileReference fileRef = IndexFileNameUtil.getIndexAbsoluteFileRef(treeOpDesc, - ctx.getTaskAttemptId().getTaskId().getPartition(), ctx.getIOManager()); - IBufferCache bufferCache = opDesc.getStorageManager().getBufferCache(ctx); - return BTreeUtils.createBTree(bufferCache, opDesc.getStorageManager().getFileMapProvider(ctx), - treeOpDesc.getTreeIndexTypeTraits(), treeOpDesc.getTreeIndexComparatorFactories(), - BTreeLeafFrameType.REGULAR_NSM, fileRef, pageManagerFactory.createPageManager(bufferCache)); - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeDataflowHelperFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeDataflowHelperFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeDataflowHelperFactory.java deleted file mode 100644 index c4e3ce7..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeDataflowHelperFactory.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hyracks.storage.am.btree.dataflow; - -import org.apache.hyracks.api.context.IHyracksTaskContext; -import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; -import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor; -import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelper; - -public class BTreeDataflowHelperFactory implements IIndexDataflowHelperFactory { - - private static final long serialVersionUID = 1L; - - private final boolean durable; - - public BTreeDataflowHelperFactory(boolean durable) { - this.durable = durable; - } - - @Override - public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, - int partition) throws HyracksDataException { - return new BTreeDataflowHelper(opDesc, ctx, partition, durable); - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeResource.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeResource.java new file mode 100644 index 0000000..90bc126 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeResource.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.storage.am.btree.dataflow; + +import org.apache.hyracks.api.application.INCServiceContext; +import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; +import org.apache.hyracks.api.dataflow.value.ITypeTraits; +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.io.FileReference; +import org.apache.hyracks.api.io.IIOManager; +import org.apache.hyracks.storage.am.btree.frames.BTreeLeafFrameType; +import org.apache.hyracks.storage.am.btree.util.BTreeUtils; +import org.apache.hyracks.storage.am.common.api.IPageManagerFactory; +import org.apache.hyracks.storage.common.IIndex; +import org.apache.hyracks.storage.common.IResource; +import org.apache.hyracks.storage.common.IStorageManager; +import org.apache.hyracks.storage.common.buffercache.IBufferCache; + +public class BTreeResource implements IResource { + + private static final long serialVersionUID = 1L; + private final String path; + private final IStorageManager storageManager; + private final ITypeTraits[] typeTraits; + private final IBinaryComparatorFactory[] comparatorFactories; + private final IPageManagerFactory pageManagerFactory; + + public BTreeResource(String path, IStorageManager storageManager, ITypeTraits[] typeTraits, + IBinaryComparatorFactory[] comparatorFactories, IPageManagerFactory pageManagerFactory) { + this.path = path; + this.storageManager = storageManager; + this.typeTraits = typeTraits; + this.comparatorFactories = comparatorFactories; + this.pageManagerFactory = pageManagerFactory; + } + + @Override + public IIndex createInstance(INCServiceContext ctx) throws HyracksDataException { + IBufferCache bufferCache = storageManager.getBufferCache(ctx); + IIOManager ioManager = ctx.getIoManager(); + FileReference resourceRef = ioManager.resolve(path); + return BTreeUtils.createBTree(bufferCache, storageManager.getFileMapProvider(ctx), typeTraits, + comparatorFactories, BTreeLeafFrameType.REGULAR_NSM, resourceRef, + pageManagerFactory.createPageManager(bufferCache)); + } + + @Override + public String getPath() { + return path; + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeResourceFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeResourceFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeResourceFactory.java new file mode 100644 index 0000000..7a21495 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeResourceFactory.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.storage.am.btree.dataflow; + +import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; +import org.apache.hyracks.api.dataflow.value.ITypeTraits; +import org.apache.hyracks.api.io.FileReference; +import org.apache.hyracks.storage.am.common.api.IPageManagerFactory; +import org.apache.hyracks.storage.common.IResource; +import org.apache.hyracks.storage.common.IResourceFactory; +import org.apache.hyracks.storage.common.IStorageManager; + +public class BTreeResourceFactory implements IResourceFactory { + + private static final long serialVersionUID = 1L; + private final IStorageManager storageManager; + private final ITypeTraits[] typeTraits; + private final IBinaryComparatorFactory[] comparatorFactories; + private final IPageManagerFactory pageManagerFactory; + + public BTreeResourceFactory(IStorageManager storageManager, ITypeTraits[] typeTraits, + IBinaryComparatorFactory[] comparatorFactories, IPageManagerFactory pageManagerFactory) { + this.storageManager = storageManager; + this.typeTraits = typeTraits; + this.comparatorFactories = comparatorFactories; + this.pageManagerFactory = pageManagerFactory; + } + + @Override + public IResource createResource(FileReference fileRef) { + return new BTreeResource(fileRef.getRelativePath(), storageManager, typeTraits, comparatorFactories, + pageManagerFactory); + } + +}
