>From Ritik Raj <[email protected]>:
Ritik Raj has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19323 )
Change subject: [ASTERIXDB-3548][STO] Avoiding assembly of object fields
however nested.
......................................................................
[ASTERIXDB-3548][STO] Avoiding assembly of object fields however nested.
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
Updated TupleProjector to evaluate the result without
assembling the record.
Patch-seq: 7
Ext-ref: MB-63032
Change-Id: I882d6ae1c4cd0a2850de281dbdcff5851bafada5
---
M
hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/projection/ITupleProjector.java
M
asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/ObjectValueAssembler.java
M
asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java
M
asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/ColumnAssembler.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
M
asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnTupleProjector.java
M
asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnWithMetaTupleProjector.java
M
asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/AssemblerBuilderVisitor.java
M
asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java
9 files changed, 138 insertions(+), 28 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/23/19323/1
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/AssemblerBuilderVisitor.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/AssemblerBuilderVisitor.java
index 2537abe..f83cdf3 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/AssemblerBuilderVisitor.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/AssemblerBuilderVisitor.java
@@ -47,6 +47,7 @@
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.ints.IntList;
@@ -58,6 +59,9 @@
private final List<AbstractPrimitiveValueAssembler> valueAssemblers;
private final IValueGetterFactory valueGetterFactory;
private final Map<Integer, IColumnValuesReader> primaryKeyReaders;
+ private final ArrayTupleBuilder tupleBuilder;
+ private final IntList insertedFieldOrder;
+ private final int requestedFieldsCount;
private AbstractValueAssembler rootAssembler;
//Recursion info
@@ -66,13 +70,16 @@
private int level;
public AssemblerBuilderVisitor(QueryColumnMetadata columnMetadata,
IColumnValuesReaderFactory readerFactory,
- IValueGetterFactory valueGetterFactory) {
+ IValueGetterFactory valueGetterFactory, int requestedFieldsCount) {
this.columnMetadata = columnMetadata;
this.readerFactory = readerFactory;
this.valueGetterFactory = valueGetterFactory;
+ this.requestedFieldsCount = requestedFieldsCount;
valueAssemblers = new ArrayList<>();
delimiters = new IntArrayList();
primaryKeyReaders = new HashMap<>();
+ tupleBuilder = new ArrayTupleBuilder(requestedFieldsCount);
+ insertedFieldOrder = new IntArrayList(requestedFieldsCount);
for (IColumnValuesReader reader :
columnMetadata.getPrimaryKeyReaders()) {
primaryKeyReaders.put(reader.getColumnIndex(), reader);
}
@@ -120,6 +127,27 @@
return objectAssembler;
}
+ public AbstractValueAssembler visit(FlatObjectSchemaNode orderedNode,
AssemblerInfo info)
+ throws HyracksDataException {
+ AddFlatBytesAssembler addByteValueAssembler =
+ new AddFlatBytesAssembler(tupleBuilder, insertedFieldOrder,
requestedFieldsCount, info);
+ int numberOfAddedChildren = 0;
+ List<AbstractSchemaNode> children = orderedNode.getChildren();
+ for (int i = 0; i < children.size(); i++) {
+ AbstractSchemaNode childNode = children.get(i);
+ numberOfAddedChildren++;
+ IAType childType = getChildType(childNode, BuiltinType.ANY);
+ //The last child should be a delegate
+ boolean delegate = numberOfAddedChildren ==
orderedNode.getChildren().size();
+ // initialize level with the one computed in schemaClipper stage.
+ level = orderedNode.getChildLevel(i);
+ AssemblerInfo childInfo =
+ new AssemblerInfo(childType, addByteValueAssembler,
delegate, childNode.getOutputIndex());
+ childNode.accept(this, childInfo);
+ }
+ return addByteValueAssembler;
+ }
+
private BitSet handleDeclaredFields(ObjectSchemaNode objectNode,
AssemblerInfo info,
ObjectValueAssembler objectAssembler) throws HyracksDataException {
ARecordType declaredType = (ARecordType) info.getDeclaredType();
@@ -233,12 +261,6 @@
return assembler;
}
- @Override
- public AbstractValueAssembler visit(FlatObjectSchemaNode flatObjectNode,
AssemblerInfo arg)
- throws HyracksDataException {
- throw new UnsupportedOperationException("FlatObjectSchemaNode is not
supported");
- }
-
private ATypeTag getTypeTag(AssemblerInfo info, PrimitiveSchemaNode
primitiveNode) {
IAType declaredType = info.getDeclaredType();
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/ObjectValueAssembler.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/ObjectValueAssembler.java
index 0ea70da..6743d68 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/ObjectValueAssembler.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/ObjectValueAssembler.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.column.assembler;
+import java.io.DataOutput;
import java.io.IOException;
import org.apache.asterix.builders.RecordBuilder;
@@ -25,6 +26,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.api.IAssembledTupleReference;
import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference;
public class ObjectValueAssembler extends AbstractNestedValueAssembler {
@@ -76,8 +78,18 @@
}
@Override
- public IAssembledTupleReference getRecord(ArrayTupleReference
tupleReference, boolean isPrevious)
- throws IOException {
+ public IAssembledTupleReference getRecord(ArrayTupleReference
tupleReference, boolean isPrevious) {
return getValue();
}
+
+ public boolean handleAssembledResult(IAssembledTupleReference tupleRef,
DataOutput dos, ArrayTupleBuilder tb)
+ throws IOException {
+ IValueReference assembledRecord = (IValueReference) tupleRef;
+ if (assembledRecord == null || assembledRecord == MISSING) {
+ return false;
+ }
+ dos.write(assembledRecord.getByteArray(),
assembledRecord.getStartOffset(), assembledRecord.getLength());
+ tb.addFieldEndOffset();
+ return true;
+ }
}
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/ColumnAssembler.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/ColumnAssembler.java
index d6e1eba..4938d3a 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/ColumnAssembler.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/ColumnAssembler.java
@@ -49,7 +49,7 @@
IColumnValuesReaderFactory readerFactory, IValueGetterFactory
valueGetterFactory, int requestedFieldsCount)
throws HyracksDataException {
AssemblerBuilderVisitor builderVisitor =
- new AssemblerBuilderVisitor(columnMetadata, readerFactory,
valueGetterFactory);
+ new AssemblerBuilderVisitor(columnMetadata, readerFactory,
valueGetterFactory, requestedFieldsCount);
assemblers = builderVisitor.createValueAssemblers(node, declaredType);
rootAssembler = (AbstractNestedValueAssembler)
builderVisitor.getRootAssembler();
tupleReference = new ArrayTupleReference();
@@ -83,7 +83,7 @@
if (tupleIndex == numberOfTuples) {
rootAssembler.end();
//return empty record
- return rootAssembler.getValue();
+ return getAssembledTupleReference(false);
}
int index = 0;
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnTupleProjector.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnTupleProjector.java
index d21e367..f07c06d 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnTupleProjector.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnTupleProjector.java
@@ -24,7 +24,6 @@
import java.io.IOException;
import java.util.Map;
-import org.apache.asterix.column.assembler.value.MissingValueGetter;
import org.apache.asterix.column.assembler.value.ValueGetterFactory;
import
org.apache.asterix.column.filter.iterable.IColumnIterableFilterEvaluatorFactory;
import
org.apache.asterix.column.filter.range.IColumnRangeFilterEvaluatorFactory;
@@ -119,22 +118,47 @@
tb.addFieldEndOffset();
}
if (isColumnar(tuple)) {
- IValueReference assembledRecord = (IValueReference)
getAssembledValue(tuple);
- if (assembledRecord == MissingValueGetter.MISSING) {
+ boolean processedResult = processedAssembledRecord(tuple, dos, tb);
+ if (!processedResult) {
return null;
}
- dos.write(assembledRecord.getByteArray(),
assembledRecord.getStartOffset(), assembledRecord.getLength());
} else {
- dos.write(tuple.getFieldData(numberOfPrimaryKeys),
tuple.getFieldStart(numberOfPrimaryKeys),
- tuple.getFieldLength(numberOfPrimaryKeys));
+ // get the below byte array and put in a voidPointable, this will
be the value of the rootEvaluator,
+ // convert voidPointable to BareTupleReference, and use it with
the scalarEvaluator
+ // to evaluate the result and store in the VoidPointable of the
corresponding evaluatorNode
+ // and use it while evaluating the childNode.
+
+ // this voidPointable should be of evaluatorTree's root.
+ if (scalarEvaluatorTree != null && rowTupleEvaluatorFactoryTree !=
ScalarEvaluatorFactoryTree.defaultTree) {
+
scalarEvaluatorTree.getRoot().resetPointable(tuple.getFieldData(numberOfPrimaryKeys),
+ tuple.getFieldStart(numberOfPrimaryKeys),
tuple.getFieldLength(numberOfPrimaryKeys));
+
+ // evaluate the evaluatorTree, and put leafs value in the
output stream.
+ scalarEvaluatorTree.evaluate(scalarEvaluatorTree.getRoot(),
tupleReference, dos, tb);
+ } else {
+ dos.write(tuple.getFieldData(numberOfPrimaryKeys),
tuple.getFieldStart(numberOfPrimaryKeys),
+ tuple.getFieldLength(numberOfPrimaryKeys));
+ tb.addFieldEndOffset();
+ }
}
- tb.addFieldEndOffset();
//Write meta (if any)
writeMeta(tuple, dos, tb);
return assembledTupleReference.reset(tb);
}
+ @Override
+ public int getFieldCount(int fieldCount) {
+ return getNumberOfTupleFields();
+ }
+
+ protected boolean processedAssembledRecord(ITupleReference tuple,
DataOutput dos, ArrayTupleBuilder tb)
+ throws IOException {
+ QueryColumnTupleReference columnTuple = (QueryColumnTupleReference)
tuple;
+ IAssembledTupleReference tupleRef = columnTuple.getAssembledValue();
+ return columnTuple.getRootAssembler().handleAssembledResult(tupleRef,
dos, tb);
+ }
+
protected ScalarEvaluatorTree createScalarEvaluatorTree(EvaluatorContext
evaluatorContext)
throws HyracksDataException {
// Iterate over rowTupleEvaluatorFactoryTree, and create a
scalarEvaluatorTree
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnWithMetaTupleProjector.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnWithMetaTupleProjector.java
index 4b48f2f..9f2fa10 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnWithMetaTupleProjector.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnWithMetaTupleProjector.java
@@ -36,6 +36,7 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.data.std.api.IAssembledTupleReference;
import org.apache.hyracks.data.std.api.IValueReference;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
@@ -93,9 +94,11 @@
}
@Override
- protected IValueReference getAssembledValue(ITupleReference tuple) throws
IOException {
+ protected boolean processedAssembledRecord(ITupleReference tuple,
DataOutput dos, ArrayTupleBuilder tb)
+ throws IOException {
QueryColumnWithMetaTupleReference columnTuple =
(QueryColumnWithMetaTupleReference) tuple;
- return (IValueReference) columnTuple.getAssembledValue();
+ IAssembledTupleReference tupleRef = columnTuple.getAssembledValue();
+ return columnTuple.getRootAssembler().handleAssembledResult(tupleRef,
dos, tb);
}
@Override
@@ -105,13 +108,23 @@
}
if (tuple instanceof QueryColumnWithMetaTupleReference) {
QueryColumnWithMetaTupleReference columnTuple =
(QueryColumnWithMetaTupleReference) tuple;
- IValueReference assembledRecord = (IValueReference)
columnTuple.getMetaAssembledValue();
- dos.write(assembledRecord.getByteArray(),
assembledRecord.getStartOffset(), assembledRecord.getLength());
+ IAssembledTupleReference metaAssembledValue =
columnTuple.getMetaAssembledValue();
+
columnTuple.getMetaRootAssembler().handleAssembledResult(metaAssembledValue,
dos, tb);
} else {
- dos.write(tuple.getFieldData(numberOfPrimaryKeys + 1),
tuple.getFieldStart(numberOfPrimaryKeys + 1),
- tuple.getFieldLength(numberOfPrimaryKeys + 1));
+ int metaRecordIndex = numberOfPrimaryKeys + 1; // 0th based
indexing.
+ if (metaScalarEvaluatorTree != null
+ && metaRowTupleEvaluatorFactoryTree !=
ScalarEvaluatorFactoryTree.defaultTree) {
+
metaScalarEvaluatorTree.getRoot().resetPointable(tuple.getFieldData(metaRecordIndex),
+ tuple.getFieldStart(metaRecordIndex),
tuple.getFieldLength(metaRecordIndex));
+
+ // evaluate the evaluatorTree, and put leafs value in the
output stream.
+
metaScalarEvaluatorTree.evaluate(metaScalarEvaluatorTree.getRoot(),
metaTupleReference, dos, tb);
+ } else {
+ dos.write(tuple.getFieldData(metaRecordIndex),
tuple.getFieldStart(metaRecordIndex),
+ tuple.getFieldLength(metaRecordIndex));
+ tb.addFieldEndOffset();
+ }
}
- tb.addFieldEndOffset();
}
@Override
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java
index b2914a0..4df9c35 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java
@@ -22,7 +22,7 @@
import java.nio.ByteBuffer;
import java.util.List;
-import org.apache.asterix.column.assembler.value.MissingValueGetter;
+import org.apache.asterix.column.assembler.AbstractNestedValueAssembler;
import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream;
import org.apache.asterix.column.bytes.stream.in.ByteBufferInputStream;
import org.apache.asterix.column.bytes.stream.in.DummyBytesInputStream;
@@ -147,7 +147,10 @@
appendExceptionInformation(e, previousIndex);
throw e;
}
+ }
+ public AbstractNestedValueAssembler getRootAssembler() {
+ return assembler.getRootAssembler();
}
private IFilterApplier createFilterApplier() {
@@ -169,6 +172,6 @@
columnFilterEvaluator.evaluate();
return assembler.nextValue();
}
- return MissingValueGetter.MISSING;
+ return null;
}
}
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java
index d3086e2..c38b92d 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java
@@ -22,6 +22,7 @@
import java.nio.ByteBuffer;
import java.util.List;
+import org.apache.asterix.column.assembler.AbstractNestedValueAssembler;
import org.apache.asterix.column.assembler.value.MissingValueGetter;
import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream;
import org.apache.asterix.column.bytes.stream.in.ByteBufferInputStream;
@@ -141,6 +142,14 @@
}
}
+ public AbstractNestedValueAssembler getRootAssembler() {
+ return assembler.getRootAssembler();
+ }
+
+ public AbstractNestedValueAssembler getMetaRootAssembler() {
+ return metaAssembler.getRootAssembler();
+ }
+
@Override
public void skip(int count) throws HyracksDataException {
metaAssembler.skip(count);
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
index bc111dc..3123d94 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
@@ -100,7 +100,7 @@
@Override
protected int getFieldCount(IIndex index) {
- return ((ITreeIndex) index).getFieldCount();
+ return tupleProjector.getFieldCount(((ITreeIndex)
index).getFieldCount());
}
@Override
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/projection/ITupleProjector.java
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/projection/ITupleProjector.java
index ba23e30..a06b608 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/projection/ITupleProjector.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/projection/ITupleProjector.java
@@ -26,4 +26,11 @@
public interface ITupleProjector {
ITupleReference project(ITupleReference tuple, DataOutput dos,
ArrayTupleBuilder tb) throws IOException;
+
+ /**
+ * @return the number of fields to be projected
+ */
+ default int getFieldCount(int fieldCount) {
+ return fieldCount;
+ }
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19323
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I882d6ae1c4cd0a2850de281dbdcff5851bafada5
Gerrit-Change-Number: 19323
Gerrit-PatchSet: 1
Gerrit-Owner: Ritik Raj <[email protected]>
Gerrit-MessageType: newchange
