>From Ali Alsuliman <[email protected]>: Ali Alsuliman has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19651 )
Change subject: [NO ISSUE][FUN] Use the correct tuple projector in query_partition() ...................................................................... [NO ISSUE][FUN] Use the correct tuple projector in query_partition() - user model changes: no - storage format changes: no - interface changes: no Details: - Update private storage-component() to include the size and component path. - Update log for finding metadata key from an index. Ext-ref: MB-66346 Change-Id: I17d777b2067ad51780b9ea740a8673acadc37906 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19651 Tested-by: Ali Alsuliman <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> Reviewed-by: Murtadha Hubail <[email protected]> Reviewed-by: Michael Blow <[email protected]> --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryPartitionDatasource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/util/ComponentUtils.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsReader.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java 4 files changed, 45 insertions(+), 9 deletions(-) Approvals: Murtadha Hubail: Looks good to me, approved Michael Blow: Looks good to me, approved Ali Alsuliman: Looks good to me, but someone else must approve; Verified diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryPartitionDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryPartitionDatasource.java index 4917b33..1101ace 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryPartitionDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryPartitionDatasource.java @@ -28,6 +28,7 @@ import org.apache.asterix.metadata.declared.FunctionDataSource; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.metadata.entities.Dataset; +import org.apache.asterix.metadata.utils.IndexUtil; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.IAType; import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; @@ -50,6 +51,7 @@ import org.apache.hyracks.api.dataflow.IOperatorDescriptor; import org.apache.hyracks.api.job.JobSpecification; import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory; +import org.apache.hyracks.storage.common.projection.ITupleProjectorFactory; public class QueryPartitionDatasource extends FunctionDataSource { @@ -111,8 +113,15 @@ ITupleFilterFactory tupleFilterFactory, long outputLimit, IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig, IProjectionFiltrationInfo projectionInfo) throws AlgebricksException { + ARecordType metaItemType = null; + if (ds.hasMetaPart()) { + metaItemType = (ARecordType) schemaTypes[2]; + } + ARecordType datasetType = (ARecordType) schemaTypes[1]; + ITupleProjectorFactory tupleProjectorFactory = IndexUtil.createTupleProjectorFactory(context, typeEnv, + ds.getDatasetFormatInfo(), projectionInfo, datasetType, metaItemType, ds.getPrimaryKeys().size()); return metadataProvider.getBtreePartitionSearchRuntime(jobSpec, opSchema, typeEnv, context, ds, - tupleFilterFactory, outputLimit, partitionNum); + tupleFilterFactory, tupleProjectorFactory, outputLimit, partitionNum); } @Override diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsReader.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsReader.java index 81f680d..65d7015 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsReader.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsReader.java @@ -33,15 +33,15 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentId; +import org.apache.hyracks.util.JSONUtil; public class StorageComponentsReader extends FunctionReader { - private final List<String> components; private final Iterator<String> it; private final CharArrayRecord record; public StorageComponentsReader(String nodeId, DatasetResource dsr) throws HyracksDataException { - components = new ArrayList<>(); + List<String> components = new ArrayList<>(); if (dsr != null && dsr.isOpen()) { Map<Long, IndexInfo> indexes = dsr.getIndexes(); StringBuilder strBuilder = new StringBuilder(); @@ -56,7 +56,7 @@ strBuilder.append("\", \"path\":\""); strBuilder.append(path); strBuilder.append("\", \"components\":["); - // syncronize over the opTracker + // synchronize over the opTracker synchronized (index.getOperationTracker()) { List<ILSMDiskComponent> diskComponents = index.getDiskComponents(); for (int i = diskComponents.size() - 1; i >= 0; i--) { @@ -70,6 +70,10 @@ strBuilder.append(id.getMinId()); strBuilder.append(",\"max\":"); strBuilder.append(id.getMaxId()); + strBuilder.append(",\"size\":"); + strBuilder.append(c.getComponentSize()); + strBuilder.append(",\"component\":"); + JSONUtil.quoteAndEscape(strBuilder, String.valueOf(c)); strBuilder.append('}'); } } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java index 66c184b..f11a338 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java @@ -169,7 +169,6 @@ import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory; import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory; -import org.apache.hyracks.storage.am.common.impls.DefaultTupleProjectorFactory; import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; import org.apache.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeBatchPointSearchOperatorDescriptor; import org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.BinaryTokenizerOperatorDescriptor; @@ -598,11 +597,11 @@ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getBtreePartitionSearchRuntime( JobSpecification jobSpec, IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv, JobGenContext context, - Dataset dataset, ITupleFilterFactory tupleFilterFactory, long outputLimit, int partitionNum) - throws AlgebricksException { + Dataset dataset, ITupleFilterFactory tupleFilterFactory, ITupleProjectorFactory tupleProjectorFactory, + long outputLimit, int partitionNum) throws AlgebricksException { return getBtreeSearchRuntime(jobSpec, opSchema, typeEnv, context, true, false, null, dataset, dataset.getDatasetName(), null, null, true, true, false, null, null, null, tupleFilterFactory, - outputLimit, false, false, DefaultTupleProjectorFactory.INSTANCE, false, partitionNum); + outputLimit, false, false, tupleProjectorFactory, false, partitionNum); } public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getBtreeSearchRuntime(JobSpecification jobSpec, diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/util/ComponentUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/util/ComponentUtils.java index a8cbfdd..bfb0c3c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/util/ComponentUtils.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/util/ComponentUtils.java @@ -93,7 +93,7 @@ // was not found in all in-memory components, search in the disk components fromDiskComponents(index, key, value); if (value.getLength() == 0) { - LOGGER.debug("{} was NOT found", key); + LOGGER.debug("{} was NOT found for index {}", key, index); } } } else { -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19651 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: ionic Gerrit-Change-Id: I17d777b2067ad51780b9ea740a8673acadc37906 Gerrit-Change-Number: 19651 Gerrit-PatchSet: 3 Gerrit-Owner: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Peeyush Gupta <[email protected]> Gerrit-Reviewer: Ritik Raj <[email protected]> Gerrit-MessageType: merged
