This is an automated email from the ASF dual-hosted git repository.
gian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 479f8fe8afa Fix ColumnarFrameCursorFactory canVectorize for virtual
columns, aggregators. (#18385)
479f8fe8afa is described below
commit 479f8fe8afa75813178f7825f882262b0ad03a6a
Author: Gian Merlino <[email protected]>
AuthorDate: Sun Aug 10 09:49:22 2025 -0700
Fix ColumnarFrameCursorFactory canVectorize for virtual columns,
aggregators. (#18385)
Previously, ColumnarFrameCursorFactory's canVectorize method didn'tc
consider virtual columns or aggregators from the CursorBuildSpec. This
meant that it would potentially return true for situations where a
virtual column or aggregator really couldn't vectorize, leading to failures
later on when the query engine attempts to create a vector cursor.
The change in RowFrameCursorFactory doesn't change any functionality,
it just structures the code more clearly by moving the CursorHolder to
an inner class instead of an anonymous class.
---
.../columnar/ColumnarFrameCursorFactory.java | 230 ++++++++------
.../frame/segment/row/RowFrameCursorFactory.java | 103 +++---
.../frame/segment/FrameCursorFactoryTest.java | 345 ++++++++++++++-------
3 files changed, 424 insertions(+), 254 deletions(-)
diff --git
a/processing/src/main/java/org/apache/druid/frame/segment/columnar/ColumnarFrameCursorFactory.java
b/processing/src/main/java/org/apache/druid/frame/segment/columnar/ColumnarFrameCursorFactory.java
index 2735ff61083..afacc53aaf8 100644
---
a/processing/src/main/java/org/apache/druid/frame/segment/columnar/ColumnarFrameCursorFactory.java
+++
b/processing/src/main/java/org/apache/druid/frame/segment/columnar/ColumnarFrameCursorFactory.java
@@ -29,9 +29,12 @@ import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.io.Closer;
import org.apache.druid.query.Order;
import org.apache.druid.query.OrderBy;
+import org.apache.druid.query.QueryContext;
+import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.vector.VectorValueMatcher;
import org.apache.druid.segment.ColumnCache;
+import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.Cursor;
import org.apache.druid.segment.CursorBuildSpec;
import org.apache.druid.segment.CursorFactory;
@@ -39,6 +42,7 @@ import org.apache.druid.segment.CursorHolder;
import org.apache.druid.segment.QueryableIndexColumnSelectorFactory;
import org.apache.druid.segment.SimpleAscendingOffset;
import org.apache.druid.segment.SimpleSettableOffset;
+import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.segment.vector.FilteredVectorOffset;
@@ -80,124 +84,156 @@ public class ColumnarFrameCursorFactory implements
CursorFactory
@Override
public CursorHolder makeCursorHolder(CursorBuildSpec spec)
{
- final Closer closer = Closer.create();
+ return new ColumnarFrameCursorHolder(spec);
+ }
+
+ @Override
+ public RowSignature getRowSignature()
+ {
+ return signature;
+ }
+
+ @Nullable
+ @Override
+ public ColumnCapabilities getColumnCapabilities(String column)
+ {
+ final int columnNumber = signature.indexOf(column);
+
+ if (columnNumber < 0) {
+ return null;
+ } else {
+ // Better than
frameReader.frameSignature().getColumnCapabilities(columnName), because this
method has more
+ // insight into what's actually going on with this column (nulls,
multivalue, etc).
+ return
columnReaders.get(columnNumber).readColumn(frame).getCapabilities();
+ }
+ }
- // Frames are not self-describing as to their sort order, so we can't
determine the sort order by looking at
- // the Frame object. We could populate this with information from the
relevant ClusterBy, but that's not available
- // at this point in the code. It could be plumbed in at some point. For
now, use an empty list.
- final List<OrderBy> ordering = Collections.emptyList();
+ private class ColumnarFrameCursorHolder implements CursorHolder
+ {
+ private final CursorBuildSpec spec;
+ private final Closer closer = Closer.create();
- return new CursorHolder()
+ private ColumnarFrameCursorHolder(CursorBuildSpec spec)
{
- @Override
- public boolean canVectorize()
- {
- return (spec.getFilter() == null ||
spec.getFilter().canVectorizeMatcher(signature))
- && spec.getVirtualColumns().canVectorize(signature);
- }
+ this.spec = spec;
+ }
- @Override
- public Cursor asCursor()
- {
- final FrameQueryableIndex index = new FrameQueryableIndex(frame,
signature, columnReaders);
- final ColumnCache columnCache = new ColumnCache(index, closer);
- final Filter filterToUse =
FrameCursorUtils.buildFilter(spec.getFilter(), spec.getInterval());
- final SimpleSettableOffset baseOffset = new
SimpleAscendingOffset(frame.numRows());
-
- final QueryableIndexColumnSelectorFactory columnSelectorFactory = new
QueryableIndexColumnSelectorFactory(
- spec.getVirtualColumns(),
- Order.NONE,
- baseOffset,
- columnCache
- );
+ /**
+ * Frames are not self-describing as to their sort order, so we can't
determine the sort order by looking at
+ * the Frame object. We could populate this with information from the
relevant ClusterBy, but that's not available
+ * at this point in the code. It could be plumbed in at some point. For
now, use an empty list.
+ */
+ private final List<OrderBy> ordering = Collections.emptyList();
- final SimpleSettableOffset offset;
- if (filterToUse == null) {
- offset = baseOffset;
- } else {
- offset = new FrameFilteredOffset(baseOffset, columnSelectorFactory,
filterToUse);
+ @Override
+ public boolean canVectorize()
+ {
+ final List<AggregatorFactory> aggregatorFactories =
spec.getAggregators();
+ final VirtualColumns virtualColumns = spec.getVirtualColumns();
+ final Filter filter = spec.getFilter();
+ final QueryContext queryContext = spec.getQueryContext();
+ final ColumnInspector inspector =
virtualColumns.wrapInspector(signature);
+
+ // Check that virtual columns are vectorizable.
+ if (!virtualColumns.isEmpty()) {
+ if
(!queryContext.getVectorizeVirtualColumns().shouldVectorize(virtualColumns.canVectorize(inspector)))
{
+ return false;
}
+ }
- return new FrameCursor(offset, columnSelectorFactory);
+ // Check that aggregators are vectorizable.
+ if (aggregatorFactories != null) {
+ for (AggregatorFactory factory : aggregatorFactories) {
+ if (!factory.canVectorize(inspector)) {
+ return false;
+ }
+ }
}
- @Override
- public List<OrderBy> getOrdering()
- {
- return ordering;
+ // Check that filter is vectorizable.
+ return filter == null || filter.canVectorizeMatcher(inspector);
+ }
+
+ @Override
+ public Cursor asCursor()
+ {
+ final FrameQueryableIndex index = new FrameQueryableIndex(frame,
signature, columnReaders);
+ final ColumnCache columnCache = new ColumnCache(index, closer);
+ final Filter filterToUse =
FrameCursorUtils.buildFilter(spec.getFilter(), spec.getInterval());
+ final SimpleSettableOffset baseOffset = new
SimpleAscendingOffset(frame.numRows());
+
+ final QueryableIndexColumnSelectorFactory columnSelectorFactory = new
QueryableIndexColumnSelectorFactory(
+ spec.getVirtualColumns(),
+ Order.NONE,
+ baseOffset,
+ columnCache
+ );
+
+ final SimpleSettableOffset offset;
+ if (filterToUse == null) {
+ offset = baseOffset;
+ } else {
+ offset = new FrameFilteredOffset(baseOffset, columnSelectorFactory,
filterToUse);
}
- @Nullable
- @Override
- public VectorCursor asVectorCursor()
- {
- if (!canVectorize()) {
- throw new ISE("Cannot vectorize. Check 'canVectorize' before calling
'asVectorCursor'.");
- }
+ return new FrameCursor(offset, columnSelectorFactory);
+ }
- final FrameQueryableIndex index = new FrameQueryableIndex(frame,
signature, columnReaders);
- final Filter filterToUse =
FrameCursorUtils.buildFilter(spec.getFilter(), spec.getInterval());
- final VectorOffset baseOffset = new NoFilterVectorOffset(
- spec.getQueryContext().getVectorSize(),
- 0,
- frame.numRows()
+ @Override
+ public List<OrderBy> getOrdering()
+ {
+ return ordering;
+ }
+
+ @Nullable
+ @Override
+ public VectorCursor asVectorCursor()
+ {
+ if (!canVectorize()) {
+ throw new ISE("Cannot vectorize. Check 'canVectorize' before calling
'asVectorCursor'.");
+ }
+
+ final FrameQueryableIndex index = new FrameQueryableIndex(frame,
signature, columnReaders);
+ final Filter filterToUse =
FrameCursorUtils.buildFilter(spec.getFilter(), spec.getInterval());
+ final VectorOffset baseOffset = new NoFilterVectorOffset(
+ spec.getQueryContext().getVectorSize(),
+ 0,
+ frame.numRows()
+ );
+ final ColumnCache columnCache = new ColumnCache(index, closer);
+
+ // baseSelectorFactory using baseOffset is the column selector for
filtering.
+ final VectorColumnSelectorFactory baseSelectorFactory = new
QueryableIndexVectorColumnSelectorFactory(
+ index,
+ baseOffset,
+ columnCache,
+ spec.getVirtualColumns()
+ );
+
+ if (filterToUse == null) {
+ return new FrameVectorCursor(baseOffset, baseSelectorFactory);
+ } else {
+ final VectorValueMatcher matcher =
filterToUse.makeVectorMatcher(baseSelectorFactory);
+ final FilteredVectorOffset filteredOffset =
FilteredVectorOffset.create(
+ baseOffset,
+ matcher
);
- final ColumnCache columnCache = new ColumnCache(index, closer);
- // baseSelectorFactory using baseOffset is the column selector for
filtering.
- final VectorColumnSelectorFactory baseSelectorFactory = new
QueryableIndexVectorColumnSelectorFactory(
+ final VectorColumnSelectorFactory filteredSelectorFactory = new
QueryableIndexVectorColumnSelectorFactory(
index,
- baseOffset,
+ filteredOffset,
columnCache,
spec.getVirtualColumns()
);
- if (filterToUse == null) {
- return new FrameVectorCursor(baseOffset, baseSelectorFactory);
- } else {
- final VectorValueMatcher matcher =
filterToUse.makeVectorMatcher(baseSelectorFactory);
- final FilteredVectorOffset filteredOffset =
FilteredVectorOffset.create(
- baseOffset,
- matcher
- );
-
- final VectorColumnSelectorFactory filteredSelectorFactory = new
QueryableIndexVectorColumnSelectorFactory(
- index,
- filteredOffset,
- columnCache,
- spec.getVirtualColumns()
- );
-
- return new FrameVectorCursor(filteredOffset,
filteredSelectorFactory);
- }
- }
-
- @Override
- public void close()
- {
- CloseableUtils.closeAndWrapExceptions(closer);
+ return new FrameVectorCursor(filteredOffset, filteredSelectorFactory);
}
- };
- }
-
- @Override
- public RowSignature getRowSignature()
- {
- return signature;
- }
-
- @Nullable
- @Override
- public ColumnCapabilities getColumnCapabilities(String column)
- {
- final int columnNumber = signature.indexOf(column);
+ }
- if (columnNumber < 0) {
- return null;
- } else {
- // Better than
frameReader.frameSignature().getColumnCapabilities(columnName), because this
method has more
- // insight into what's actually going on with this column (nulls,
multivalue, etc).
- return
columnReaders.get(columnNumber).readColumn(frame).getCapabilities();
+ @Override
+ public void close()
+ {
+ CloseableUtils.closeAndWrapExceptions(closer);
}
}
}
diff --git
a/processing/src/main/java/org/apache/druid/frame/segment/row/RowFrameCursorFactory.java
b/processing/src/main/java/org/apache/druid/frame/segment/row/RowFrameCursorFactory.java
index 74fa84462cd..93523893c8c 100644
---
a/processing/src/main/java/org/apache/druid/frame/segment/row/RowFrameCursorFactory.java
+++
b/processing/src/main/java/org/apache/druid/frame/segment/row/RowFrameCursorFactory.java
@@ -69,52 +69,7 @@ public class RowFrameCursorFactory implements CursorFactory
@Override
public CursorHolder makeCursorHolder(CursorBuildSpec spec)
{
- // Frames are not self-describing as to their sort order, so we can't
determine the sort order by looking at
- // the Frame object. We could populate this with information from the
relevant ClusterBy, but that's not available
- // at this point in the code. It could be plumbed in at some point. For
now, use an empty list.
- final List<OrderBy> ordering = Collections.emptyList();
-
- return new CursorHolder()
- {
- @Nullable
- @Override
- public Cursor asCursor()
- {
- final Filter filterToUse =
FrameCursorUtils.buildFilter(spec.getFilter(), spec.getInterval());
-
- final SimpleSettableOffset baseOffset = new
SimpleAscendingOffset(frame.numRows());
-
- final ColumnSelectorFactory columnSelectorFactory =
- spec.getVirtualColumns().wrap(
- new FrameColumnSelectorFactory(
- frame,
- frameReader.signature(),
- fieldReaders,
- new CursorFrameRowPointer(frame, baseOffset)
- )
- );
-
- final SimpleSettableOffset offset;
- if (filterToUse == null) {
- offset = baseOffset;
- } else {
- offset = new FrameFilteredOffset(baseOffset, columnSelectorFactory,
filterToUse);
- }
-
- final FrameCursor cursor = new FrameCursor(offset,
columnSelectorFactory);
-
- // Note: if anything closeable is ever added to this Sequence, make
sure to update FrameProcessors.makeCursor.
- // Currently, it assumes that closing the Sequence does nothing.
- return cursor;
- }
-
- @Nullable
- @Override
- public List<OrderBy> getOrdering()
- {
- return ordering;
- }
- };
+ return new RowFrameCursorHolder(spec);
}
@Override
@@ -129,4 +84,60 @@ public class RowFrameCursorFactory implements CursorFactory
{
return frameReader.signature().getColumnCapabilities(column);
}
+
+ private class RowFrameCursorHolder implements CursorHolder
+ {
+ private final CursorBuildSpec spec;
+
+ /**
+ * Frames are not self-describing as to their sort order, so we can't
determine the sort order by looking at
+ * the Frame object. We could populate this with information from the
relevant ClusterBy, but that's not available
+ * at this point in the code. It could be plumbed in at some point. For
now, use an empty list.
+ */
+ private final List<OrderBy> ordering = Collections.emptyList();
+
+ private RowFrameCursorHolder(CursorBuildSpec spec)
+ {
+ this.spec = spec;
+ }
+
+ @Nullable
+ @Override
+ public Cursor asCursor()
+ {
+ final Filter filterToUse =
FrameCursorUtils.buildFilter(spec.getFilter(), spec.getInterval());
+
+ final SimpleSettableOffset baseOffset = new
SimpleAscendingOffset(frame.numRows());
+
+ final ColumnSelectorFactory columnSelectorFactory =
+ spec.getVirtualColumns().wrap(
+ new FrameColumnSelectorFactory(
+ frame,
+ frameReader.signature(),
+ fieldReaders,
+ new CursorFrameRowPointer(frame, baseOffset)
+ )
+ );
+
+ final SimpleSettableOffset offset;
+ if (filterToUse == null) {
+ offset = baseOffset;
+ } else {
+ offset = new FrameFilteredOffset(baseOffset, columnSelectorFactory,
filterToUse);
+ }
+
+ final FrameCursor cursor = new FrameCursor(offset,
columnSelectorFactory);
+
+ // Note: if anything closeable is ever added to this Sequence, make sure
to update FrameProcessors.makeCursor.
+ // Currently, it assumes that closing the Sequence does nothing.
+ return cursor;
+ }
+
+ @Nullable
+ @Override
+ public List<OrderBy> getOrdering()
+ {
+ return ordering;
+ }
+ }
}
diff --git
a/processing/src/test/java/org/apache/druid/frame/segment/FrameCursorFactoryTest.java
b/processing/src/test/java/org/apache/druid/frame/segment/FrameCursorFactoryTest.java
index 947e4c5cfa8..3ac80cd99a8 100644
---
a/processing/src/test/java/org/apache/druid/frame/segment/FrameCursorFactoryTest.java
+++
b/processing/src/test/java/org/apache/druid/frame/segment/FrameCursorFactoryTest.java
@@ -19,21 +19,18 @@
package org.apache.druid.frame.segment;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.apache.druid.frame.FrameType;
import org.apache.druid.frame.testutil.FrameTestUtil;
import org.apache.druid.java.util.common.Intervals;
-import org.apache.druid.java.util.common.Pair;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.guava.Sequence;
import org.apache.druid.math.expr.ExprMacroTable;
import org.apache.druid.query.QueryContext;
import org.apache.druid.query.QueryContexts;
+import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory;
import org.apache.druid.query.extraction.TimeFormatExtractionFn;
import org.apache.druid.query.extraction.UpperExtractionFn;
-import org.apache.druid.query.filter.DimFilter;
-import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.SelectorDimFilter;
import org.apache.druid.segment.Cursor;
import org.apache.druid.segment.CursorBuildSpec;
@@ -48,27 +45,21 @@ import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.segment.column.ValueType;
-import org.apache.druid.segment.filter.Filters;
import org.apache.druid.segment.vector.VectorCursor;
import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
import org.apache.druid.testing.InitializedNullHandlingTest;
-import org.hamcrest.CoreMatchers;
import org.joda.time.Interval;
import org.junit.After;
import org.junit.Assert;
-import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import javax.annotation.Nullable;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
-import java.util.function.Function;
public class FrameCursorFactoryTest
{
@@ -173,11 +164,7 @@ public class FrameCursorFactoryTest
private static final int VECTOR_SIZE = 7;
private final FrameType frameType;
- @Nullable
- private final Filter filter;
private final Interval interval;
- private final VirtualColumns virtualColumns;
- private final boolean descending;
private CursorFactory queryableCursorFactory;
private FrameSegment frameSegment;
@@ -186,93 +173,33 @@ public class FrameCursorFactoryTest
ImmutableMap.of(QueryContexts.VECTOR_SIZE_KEY, VECTOR_SIZE)
);
- private CursorBuildSpec buildSpec;
-
public CursorTests(
FrameType frameType,
- @Nullable DimFilter filter,
- Interval interval,
- VirtualColumns virtualColumns,
- boolean descending
+ Interval interval
)
{
this.frameType = frameType;
- this.filter = Filters.toFilter(filter);
this.interval = interval;
- this.virtualColumns = virtualColumns;
- this.descending = descending;
- this.buildSpec =
- CursorBuildSpec.builder()
- .setFilter(this.filter)
- .setInterval(this.interval)
- .setVirtualColumns(this.virtualColumns)
- .setPreferredOrdering(descending ?
Cursors.descendingTimeOrder() : Collections.emptyList())
- .setQueryContext(queryContext)
- .build();
}
- @Parameterized.Parameters(name = "frameType = {0}, "
- + "filter = {1}, "
- + "interval = {2}, "
- + "virtualColumns = {3}, "
- + "descending = {4}")
+ @Parameterized.Parameters(
+ name = "frameType = {0}, interval = {1}"
+ )
public static Iterable<Object[]> constructorFeeder()
{
final List<Object[]> constructors = new ArrayList<>();
- final List<Interval> intervals = Arrays.asList(
+ final List<Interval> intervals = List.of(
TestIndex.getMMappedTestIndex().getDataInterval(),
Intervals.ETERNITY,
Intervals.of("2011-04-01T00:00:00.000Z/2011-04-02T00:00:00.000Z"),
Intervals.of("3001/3002")
);
- final List<Pair<DimFilter, VirtualColumns>> filtersAndVirtualColumns =
new ArrayList<>();
- filtersAndVirtualColumns.add(Pair.of(null, VirtualColumns.EMPTY));
- filtersAndVirtualColumns.add(Pair.of(new SelectorDimFilter("quality",
"automotive", null), VirtualColumns.EMPTY));
- filtersAndVirtualColumns.add(Pair.of(
- new SelectorDimFilter("expr", "1401", null),
- VirtualColumns.create(
- ImmutableList.of(
- new ExpressionVirtualColumn(
- "expr",
- "qualityLong + 1",
- ColumnType.LONG,
- ExprMacroTable.nil()
- )
- )
- )
- ));
- filtersAndVirtualColumns.add(Pair.of(new
SelectorDimFilter("qualityLong", "1400", null), VirtualColumns.EMPTY));
- filtersAndVirtualColumns.add(Pair.of(
- new SelectorDimFilter("quality", "automotive", new
UpperExtractionFn(null)),
- VirtualColumns.EMPTY
- ));
- filtersAndVirtualColumns.add(Pair.of(new SelectorDimFilter(
- ColumnHolder.TIME_COLUMN_NAME,
- "Friday",
- new TimeFormatExtractionFn("EEEE", null, null, null, false)
- ), VirtualColumns.EMPTY));
- filtersAndVirtualColumns.add(Pair.of(new SelectorDimFilter(
- ColumnHolder.TIME_COLUMN_NAME,
- "Friday",
- new TimeFormatExtractionFn("EEEE", null, null, null, false)
- ), VirtualColumns.EMPTY));
-
for (FrameType frameType : FrameType.values()) {
- for (Pair<DimFilter, VirtualColumns> filterVirtualColumnsPair :
filtersAndVirtualColumns) {
- for (Interval interval : intervals) {
- for (boolean descending : Arrays.asList(false, true)) {
- constructors.add(
- new Object[]{
- frameType,
- filterVirtualColumnsPair.lhs,
- interval,
- filterVirtualColumnsPair.rhs,
- descending
- }
- );
- }
- }
+ for (Interval interval : intervals) {
+ constructors.add(
+ new Object[]{frameType, interval}
+ );
}
}
@@ -295,48 +222,244 @@ public class FrameCursorFactoryTest
}
}
+
+ @Test
+ public void test_fullScan()
+ {
+ final CursorBuildSpec cursorSpec =
+ CursorBuildSpec.builder()
+ .setInterval(interval)
+ .setQueryContext(queryContext)
+ .build();
+
+ verifyCursorFactory(cursorSpec, frameType.isColumnar());
+ }
+
+ @Test
+ public void test_fullScan_preferAscTime()
+ {
+ final CursorBuildSpec cursorSpec =
+ CursorBuildSpec.builder()
+ .setInterval(interval)
+ .setPreferredOrdering(Cursors.ascendingTimeOrder())
+ .setQueryContext(queryContext)
+ .build();
+
+ verifyCursorFactory(cursorSpec, frameType.isColumnar());
+ }
+
+ @Test
+ public void test_fullScan_preferDescTime()
+ {
+ final CursorBuildSpec cursorSpec =
+ CursorBuildSpec.builder()
+ .setInterval(interval)
+ .setPreferredOrdering(Cursors.descendingTimeOrder())
+ .setQueryContext(queryContext)
+ .build();
+
+ verifyCursorFactory(cursorSpec, frameType.isColumnar());
+ }
+
+ @Test
+ public void test_selectorFilter_stringColumn()
+ {
+ final CursorBuildSpec cursorSpec =
+ CursorBuildSpec.builder()
+ .setInterval(interval)
+ .setFilter(new SelectorDimFilter("quality",
"automotive", null).toFilter())
+ .setQueryContext(queryContext)
+ .build();
+
+ verifyCursorFactory(cursorSpec, frameType.isColumnar());
+ }
+
+ @Test
+ public void test_selectorFilter_numericColumn()
+ {
+ final CursorBuildSpec cursorSpec =
+ CursorBuildSpec.builder()
+ .setInterval(interval)
+ .setFilter(new SelectorDimFilter("qualityLong",
"1400", null).toFilter())
+ .setQueryContext(queryContext)
+ .build();
+
+ verifyCursorFactory(cursorSpec, frameType.isColumnar());
+ }
+
@Test
- public void test_makeCursor()
+ public void test_selectorFilter_expr()
{
- final RowSignature signature = frameCursorFactory.getRowSignature();
+ final CursorBuildSpec cursorSpec =
+ CursorBuildSpec.builder()
+ .setInterval(interval)
+ .setFilter(new SelectorDimFilter("quality",
"automotive", null).toFilter())
+ .setVirtualColumns(
+ VirtualColumns.create(
+ new ExpressionVirtualColumn(
+ "expr",
+ "qualityLong + 1",
+ ColumnType.LONG,
+ ExprMacroTable.nil()
+ )
+ )
+ )
+ .setQueryContext(queryContext)
+ .build();
+
+ verifyCursorFactory(cursorSpec, frameType.isColumnar());
+ }
+
+ @Test
+ public void test_selectorFilter_extractionFn()
+ {
+ final CursorBuildSpec cursorSpec =
+ CursorBuildSpec.builder()
+ .setInterval(interval)
+ .setFilter(
+ new SelectorDimFilter(
+ "quality",
+ "automotive",
+ new UpperExtractionFn(null)
+ ).toFilter()
+ )
+ .setQueryContext(queryContext)
+ .build();
+
+ verifyCursorFactory(cursorSpec, frameType.isColumnar());
+ }
+
+ @Test
+ public void test_selectorFilter_timeExtractionFn()
+ {
+ final CursorBuildSpec cursorSpec =
+ CursorBuildSpec.builder()
+ .setInterval(interval)
+ .setFilter(
+ new SelectorDimFilter(
+ ColumnHolder.TIME_COLUMN_NAME,
+ "Friday",
+ new TimeFormatExtractionFn("EEEE", null,
null, null, false)
+ ).toFilter()
+ )
+ .setQueryContext(queryContext)
+ .build();
+
+ verifyCursorFactory(cursorSpec, frameType.isColumnar());
+ }
+
+ @Test
+ public void test_expr_cannotVectorize()
+ {
+ final CursorBuildSpec cursorSpec =
+ CursorBuildSpec.builder()
+ .setInterval(interval)
+ .setVirtualColumns(
+ VirtualColumns.create(
+ new ExpressionVirtualColumn(
+ "expr",
+ "substring(quality, 1, 2)",
+ ColumnType.STRING,
+ ExprMacroTable.nil()
+ )
+ )
+ )
+ .setQueryContext(queryContext)
+ .build();
+
+ verifyCursorFactory(cursorSpec, false);
+ }
+
+ @Test
+ public void test_aggregator_canVectorize()
+ {
+ // Test that when an aggregator can vectorize, canVectorize returns
true. This test is not actually testing
+ // that the aggregator *works*, because verifyCursorFactory doesn't try
to use it.
+ final CursorBuildSpec cursorSpec =
+ CursorBuildSpec.builder()
+ .setInterval(interval)
+ .setAggregators(List.of(new
DoubleSumAggregatorFactory("qualitySum", "qualityLong")))
+ .setQueryContext(queryContext)
+ .build();
+
+ verifyCursorFactory(cursorSpec, frameType.isColumnar());
+ }
+
+ @Test
+ public void test_aggregator_cannotVectorize()
+ {
+ // Test that when an aggregator cannot vectorize, canVectorize returns
false. This test is not actually testing
+ // that the aggregator *works*, because verifyCursorFactory doesn't try
to use it.
+ final CursorBuildSpec cursorSpec =
+ CursorBuildSpec.builder()
+ .setInterval(interval)
+ .setAggregators(List.of(new
DoubleSumAggregatorFactory("qualitySum", "quality")))
+ .setQueryContext(queryContext)
+ .build();
+
+ verifyCursorFactory(cursorSpec, false);
+ }
+
+ /**
+ * Verify that the cursors (both vector and nonvector) from {@link
#frameCursorFactory} and
+ * {@link #queryableCursorFactory} match.
+ */
+ private void verifyCursorFactory(final CursorBuildSpec cursorSpec, final
boolean expectedCanVectorize)
+ {
+ Assert.assertEquals(
+ "expected interval (if this assertion fails, the test is likely
written incorrectly)",
+ this.interval,
+ cursorSpec.getInterval()
+ );
// Frame adapters don't know the order of the underlying frames, so they
should ignore the "preferred ordering"
// of the cursor build spec. We test this by passing the frameAdapter a
build spec with a preferred ordering,
// and passing the queryableAdapter the same build spec *without* a
preferred ordering, and verifying they match.
- final CursorBuildSpec queryableBuildSpec =
-
CursorBuildSpec.builder(buildSpec).setPreferredOrdering(Collections.emptyList()).build();
-
- try (final CursorHolder queryableCursorHolder =
queryableCursorFactory.makeCursorHolder(queryableBuildSpec);
- final CursorHolder frameCursorHolder =
frameCursorFactory.makeCursorHolder(buildSpec)) {
- final Sequence<List<Object>> queryableRows =
-
FrameTestUtil.readRowsFromCursor(advanceAndReset(queryableCursorHolder.asCursor()),
signature);
- final Sequence<List<Object>> frameRows =
-
FrameTestUtil.readRowsFromCursor(advanceAndReset(frameCursorHolder.asCursor()),
signature);
- FrameTestUtil.assertRowsEqual(queryableRows, frameRows);
+ final CursorBuildSpec queryableCursorSpec =
+
CursorBuildSpec.builder(cursorSpec).setPreferredOrdering(Collections.emptyList()).build();
+
+ try (final CursorHolder queryableCursorHolder =
queryableCursorFactory.makeCursorHolder(queryableCursorSpec);
+ final CursorHolder frameCursorHolder =
frameCursorFactory.makeCursorHolder(cursorSpec)) {
+ // Frames don't know their own order, so they cannot guarantee any
particular ordering.
+ Assert.assertEquals("ordering", Collections.emptyList(),
frameCursorHolder.getOrdering());
+ Assert.assertEquals("canVectorize", expectedCanVectorize,
frameCursorHolder.canVectorize());
+ verifyCursors(queryableCursorHolder, frameCursorHolder,
frameCursorFactory.getRowSignature());
+
+ if (expectedCanVectorize) {
+ verifyVectorCursors(queryableCursorHolder, frameCursorHolder,
frameCursorFactory.getRowSignature());
+ }
}
}
- @Test
- public void test_makeVectorCursor()
+ /**
+ * Verify that the non-vector cursors from two {@link CursorHolder} return
equivalent results.
+ */
+ private static void verifyCursors(
+ final CursorHolder queryableCursorHolder,
+ final CursorHolder frameCursorHolder,
+ final RowSignature signature
+ )
{
- // Conditions for frames to be vectorizable.
- Assume.assumeThat(frameType,
CoreMatchers.equalTo(FrameType.latestColumnar()));
- Assume.assumeFalse(descending);
- assertVectorCursorsMatch(cursorFactory ->
cursorFactory.makeCursorHolder(buildSpec));
+ final Sequence<List<Object>> queryableRows =
+
FrameTestUtil.readRowsFromCursor(advanceAndReset(queryableCursorHolder.asCursor()),
signature);
+ final Sequence<List<Object>> frameRows =
+
FrameTestUtil.readRowsFromCursor(advanceAndReset(frameCursorHolder.asCursor()),
signature);
+ FrameTestUtil.assertRowsEqual(queryableRows, frameRows);
}
- private void assertVectorCursorsMatch(final Function<CursorFactory,
CursorHolder> call)
+ /**
+ * Verify that the vector cursors from two {@link CursorHolder} return
equivalent results. Only call this
+ * if the holders have {@link CursorHolder#canVectorize()}.
+ */
+ private static void verifyVectorCursors(
+ final CursorHolder queryableCursorHolder,
+ final CursorHolder frameCursorHolder,
+ final RowSignature signature
+ )
{
- final CursorHolder cursorHolder = call.apply(queryableCursorFactory);
- final CursorHolder frameCursorHolder = call.apply(frameCursorFactory);
-
- Assert.assertTrue("queryable cursor is vectorizable",
cursorHolder.canVectorize());
- Assert.assertTrue("frame cursor is vectorizable",
frameCursorHolder.canVectorize());
-
- final RowSignature signature = frameCursorFactory.getRowSignature();
final Sequence<List<Object>> queryableRows =
-
FrameTestUtil.readRowsFromVectorCursor(advanceAndReset(cursorHolder.asVectorCursor()),
signature)
- .withBaggage(cursorHolder);
+
FrameTestUtil.readRowsFromVectorCursor(advanceAndReset(queryableCursorHolder.asVectorCursor()),
signature)
+ .withBaggage(queryableCursorHolder);
final Sequence<List<Object>> frameRows =
FrameTestUtil.readRowsFromVectorCursor(advanceAndReset(frameCursorHolder.asVectorCursor()),
signature)
.withBaggage(frameCursorHolder);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
