This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 03511e980e [feature] [null support # 7] Support null in least and
greatest transform functions. (#10575)
03511e980e is described below
commit 03511e980ea7698fe8a95a19a3e78b2bda85998c
Author: Yao Liu <[email protected]>
AuthorDate: Fri Apr 21 23:50:07 2023 -0700
[feature] [null support # 7] Support null in least and greatest transform
functions. (#10575)
---
.../function/GreatestTransformFunction.java | 200 ++++++++++++++++++++-
.../transform/function/LeastTransformFunction.java | 194 +++++++++++++++++++-
.../SelectTupleElementTransformFunction.java | 26 ++-
.../TupleSelectionTransformFunctionsTest.java | 120 +++++++++++++
4 files changed, 536 insertions(+), 4 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/GreatestTransformFunction.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/GreatestTransformFunction.java
index 9a515d9ca8..32217bd40d 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/GreatestTransformFunction.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/GreatestTransformFunction.java
@@ -19,10 +19,22 @@
package org.apache.pinot.core.operator.transform.function;
import java.math.BigDecimal;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.common.function.TransformFunctionType;
import org.apache.pinot.core.operator.blocks.ValueBlock;
+import org.roaringbitmap.RoaringBitmap;
-
+/**
+ * The <code>GreatestTransformFunction</code> implements the Greatest operator.
+ *
+ * Return the greatest results for the arguments
+ *
+ * Expected result:
+ * greatest(columnA, columnB, columnC): largest among columnA, columnB, columnC
+ *
+ * Note that null values will be ignored for evaluation. If all values are
null, we return null.
+ */
public class GreatestTransformFunction extends
SelectTupleElementTransformFunction {
public GreatestTransformFunction() {
@@ -44,6 +56,37 @@ public class GreatestTransformFunction extends
SelectTupleElementTransformFuncti
return _intValuesSV;
}
+ @Override
+ public Pair<int[], RoaringBitmap> transformToIntValuesSVWithNull(ValueBlock
valueBlock) {
+ int numDocs = valueBlock.getNumDocs();
+ initIntValuesSV(numDocs);
+ Pair<int[], RoaringBitmap> values =
_arguments.get(0).transformToIntValuesSVWithNull(valueBlock);
+ int[] curValues = values.getLeft();
+ System.arraycopy(curValues, 0, _intValuesSV, 0, numDocs);
+ RoaringBitmap nullBitmap = values.getRight();
+ for (int i = 1; i < _arguments.size(); i++) {
+ values = _arguments.get(i).transformToIntValuesSVWithNull(valueBlock);
+ RoaringBitmap curNull = values.getRight();
+ for (int j = 0; j < numDocs & j < values.getLeft().length; j++) {
+ // If current value is not null, we process the data.
+ if (curNull == null || !curNull.contains(j)) {
+ // If existing maximum value is null, we set the value directly.
+ if (nullBitmap != null && nullBitmap.contains(j)) {
+ _intValuesSV[j] = values.getLeft()[j];
+ } else {
+ _intValuesSV[j] = Math.max(_intValuesSV[j], values.getLeft()[j]);
+ }
+ }
+ }
+ if (nullBitmap != null && curNull != null) {
+ nullBitmap.and(curNull);
+ } else {
+ nullBitmap = null;
+ }
+ }
+ return ImmutablePair.of(_intValuesSV, nullBitmap);
+ }
+
@Override
public long[] transformToLongValuesSV(ValueBlock valueBlock) {
int numDocs = valueBlock.getNumDocs();
@@ -59,6 +102,37 @@ public class GreatestTransformFunction extends
SelectTupleElementTransformFuncti
return _longValuesSV;
}
+ @Override
+ public Pair<long[], RoaringBitmap>
transformToLongValuesSVWithNull(ValueBlock valueBlock) {
+ int numDocs = valueBlock.getNumDocs();
+ initLongValuesSV(numDocs);
+ Pair<long[], RoaringBitmap> values =
_arguments.get(0).transformToLongValuesSVWithNull(valueBlock);
+ long[] curValues = values.getLeft();
+ System.arraycopy(curValues, 0, _longValuesSV, 0, numDocs);
+ RoaringBitmap nullBitmap = values.getRight();
+ for (int i = 1; i < _arguments.size(); i++) {
+ values = _arguments.get(i).transformToLongValuesSVWithNull(valueBlock);
+ RoaringBitmap curNull = values.getRight();
+ for (int j = 0; j < numDocs & j < values.getLeft().length; j++) {
+ // If current value is not null, we process the data.
+ if (curNull == null || !curNull.contains(j)) {
+ // If existing maximum value is null, we set the value directly.
+ if (nullBitmap != null && nullBitmap.contains(j)) {
+ _longValuesSV[j] = values.getLeft()[j];
+ } else {
+ _longValuesSV[j] = Math.max(_longValuesSV[j], values.getLeft()[j]);
+ }
+ }
+ }
+ if (nullBitmap != null && curNull != null) {
+ nullBitmap.and(curNull);
+ } else {
+ nullBitmap = null;
+ }
+ }
+ return ImmutablePair.of(_longValuesSV, nullBitmap);
+ }
+
@Override
public float[] transformToFloatValuesSV(ValueBlock valueBlock) {
int numDocs = valueBlock.getNumDocs();
@@ -74,6 +148,37 @@ public class GreatestTransformFunction extends
SelectTupleElementTransformFuncti
return _floatValuesSV;
}
+ @Override
+ public Pair<float[], RoaringBitmap>
transformToFloatValuesSVWithNull(ValueBlock valueBlock) {
+ int numDocs = valueBlock.getNumDocs();
+ initFloatValuesSV(numDocs);
+ Pair<float[], RoaringBitmap> values =
_arguments.get(0).transformToFloatValuesSVWithNull(valueBlock);
+ float[] curValues = values.getLeft();
+ System.arraycopy(curValues, 0, _floatValuesSV, 0, numDocs);
+ RoaringBitmap nullBitmap = values.getRight();
+ for (int i = 1; i < _arguments.size(); i++) {
+ values = _arguments.get(i).transformToFloatValuesSVWithNull(valueBlock);
+ RoaringBitmap curNull = values.getRight();
+ for (int j = 0; j < numDocs & j < values.getLeft().length; j++) {
+ // If current value is not null, we process the data.
+ if (curNull != null || !curNull.contains(j)) {
+ // If existing maximum value is null, we set the value directly.
+ if (nullBitmap != null && nullBitmap.contains(j)) {
+ _floatValuesSV[j] = values.getLeft()[j];
+ } else {
+ _floatValuesSV[j] = Math.max(_floatValuesSV[j],
values.getLeft()[j]);
+ }
+ }
+ }
+ if (nullBitmap != null && curNull != null) {
+ nullBitmap.and(curNull);
+ } else {
+ nullBitmap = null;
+ }
+ }
+ return ImmutablePair.of(_floatValuesSV, nullBitmap);
+ }
+
@Override
public double[] transformToDoubleValuesSV(ValueBlock valueBlock) {
int numDocs = valueBlock.getNumDocs();
@@ -89,6 +194,37 @@ public class GreatestTransformFunction extends
SelectTupleElementTransformFuncti
return _doubleValuesSV;
}
+ @Override
+ public Pair<double[], RoaringBitmap>
transformToDoubleValuesSVWithNull(ValueBlock valueBlock) {
+ int numDocs = valueBlock.getNumDocs();
+ initDoubleValuesSV(numDocs);
+ Pair<double[], RoaringBitmap> values =
_arguments.get(0).transformToDoubleValuesSVWithNull(valueBlock);
+ double[] curValues = values.getLeft();
+ System.arraycopy(curValues, 0, _doubleValuesSV, 0, numDocs);
+ RoaringBitmap nullBitmap = values.getRight();
+ for (int i = 1; i < _arguments.size(); i++) {
+ values = _arguments.get(i).transformToDoubleValuesSVWithNull(valueBlock);
+ RoaringBitmap curNull = values.getRight();
+ for (int j = 0; j < numDocs & j < values.getLeft().length; j++) {
+ // If current value is not null, we process the data.
+ if (curNull == null || !curNull.contains(j)) {
+ // If existing maximum value is null, we set the value directly.
+ if (nullBitmap != null && nullBitmap.contains(j)) {
+ _doubleValuesSV[j] = values.getLeft()[j];
+ } else {
+ _doubleValuesSV[j] = Math.max(_doubleValuesSV[j],
values.getLeft()[j]);
+ }
+ }
+ }
+ if (nullBitmap != null && curNull != null) {
+ nullBitmap.and(curNull);
+ } else {
+ nullBitmap = null;
+ }
+ }
+ return ImmutablePair.of(_doubleValuesSV, nullBitmap);
+ }
+
@Override
public BigDecimal[] transformToBigDecimalValuesSV(ValueBlock valueBlock) {
int numDocs = valueBlock.getNumDocs();
@@ -104,6 +240,37 @@ public class GreatestTransformFunction extends
SelectTupleElementTransformFuncti
return _bigDecimalValuesSV;
}
+ @Override
+ public Pair<BigDecimal[], RoaringBitmap>
transformToBigDecimalValuesSVWithNull(ValueBlock valueBlock) {
+ int numDocs = valueBlock.getNumDocs();
+ initBigDecimalValuesSV(numDocs);
+ Pair<BigDecimal[], RoaringBitmap> values =
_arguments.get(0).transformToBigDecimalValuesSVWithNull(valueBlock);
+ BigDecimal[] curValues = values.getLeft();
+ System.arraycopy(curValues, 0, _bigDecimalValuesSV, 0, numDocs);
+ RoaringBitmap nullBitmap = values.getRight();
+ for (int i = 1; i < _arguments.size(); i++) {
+ values =
_arguments.get(i).transformToBigDecimalValuesSVWithNull(valueBlock);
+ RoaringBitmap curNull = values.getRight();
+ for (int j = 0; j < numDocs & j < values.getLeft().length; j++) {
+ // If current value is not null, we process the data.
+ if (curNull == null || !curNull.contains(j)) {
+ // If existing maximum value is null, we set the value directly.
+ if (nullBitmap != null && nullBitmap.contains(j)) {
+ _bigDecimalValuesSV[j] = values.getLeft()[j];
+ } else {
+ _bigDecimalValuesSV[j] =
_bigDecimalValuesSV[j].max(values.getLeft()[j]);
+ }
+ }
+ }
+ if (nullBitmap != null && curNull != null) {
+ nullBitmap.and(curNull);
+ } else {
+ nullBitmap = null;
+ }
+ }
+ return ImmutablePair.of(_bigDecimalValuesSV, nullBitmap);
+ }
+
@Override
public String[] transformToStringValuesSV(ValueBlock valueBlock) {
int numDocs = valueBlock.getNumDocs();
@@ -120,4 +287,35 @@ public class GreatestTransformFunction extends
SelectTupleElementTransformFuncti
}
return _stringValuesSV;
}
+
+ @Override
+ public Pair<String[], RoaringBitmap>
transformToStringValuesSVWithNull(ValueBlock valueBlock) {
+ int numDocs = valueBlock.getNumDocs();
+ initStringValuesSV(numDocs);
+ Pair<String[], RoaringBitmap> values =
_arguments.get(0).transformToStringValuesSVWithNull(valueBlock);
+ String[] curValues = values.getLeft();
+ System.arraycopy(curValues, 0, _stringValuesSV, 0, numDocs);
+ RoaringBitmap nullBitmap = values.getRight();
+ for (int i = 1; i < _arguments.size(); i++) {
+ values = _arguments.get(i).transformToStringValuesSVWithNull(valueBlock);
+ RoaringBitmap curNull = values.getRight();
+ for (int j = 0; j < numDocs & j < values.getLeft().length; j++) {
+ // If current value is not null, we process the data.
+ if (curNull == null || !curNull.contains(j)) {
+ // If existing maximum value is null, we set the value directly.
+ if (nullBitmap != null && nullBitmap.contains(j)) {
+ _stringValuesSV[j] = values.getLeft()[j];
+ } else if (_stringValuesSV[j].compareTo(values.getLeft()[j]) < 0) {
+ _stringValuesSV[j] = values.getLeft()[j];
+ }
+ }
+ }
+ if (nullBitmap != null && curNull != null) {
+ nullBitmap.and(curNull);
+ } else {
+ nullBitmap = null;
+ }
+ }
+ return ImmutablePair.of(_stringValuesSV, nullBitmap);
+ }
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LeastTransformFunction.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LeastTransformFunction.java
index 8891e2928b..4933266f06 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LeastTransformFunction.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LeastTransformFunction.java
@@ -19,10 +19,22 @@
package org.apache.pinot.core.operator.transform.function;
import java.math.BigDecimal;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.common.function.TransformFunctionType;
import org.apache.pinot.core.operator.blocks.ValueBlock;
+import org.roaringbitmap.RoaringBitmap;
-
+/**
+ * The <code>LeastTransformFunction</code> implements the Least operator.
+ *
+ * Return the smallest results for the arguments
+ *
+ * Expected result:
+ * Least(columnA, columnB, columnC): smallest among columnA, columnB, columnC
+ *
+ * Note that null values will be ignored for evaluation. If all values are
null, we return null.
+ */
public class LeastTransformFunction extends
SelectTupleElementTransformFunction {
public LeastTransformFunction() {
@@ -44,6 +56,36 @@ public class LeastTransformFunction extends
SelectTupleElementTransformFunction
return _intValuesSV;
}
+ @Override
+ public Pair<int[], RoaringBitmap> transformToIntValuesSVWithNull(ValueBlock
valueBlock) {
+ int numDocs = valueBlock.getNumDocs();
+ initIntValuesSV(numDocs);
+ Pair<int[], RoaringBitmap> values =
_arguments.get(0).transformToIntValuesSVWithNull(valueBlock);
+ System.arraycopy(values.getLeft(), 0, _intValuesSV, 0, numDocs);
+ RoaringBitmap nullBitmap = values.getRight();
+ for (int i = 1; i < _arguments.size(); i++) {
+ values = _arguments.get(i).transformToIntValuesSVWithNull(valueBlock);
+ RoaringBitmap curNull = values.getRight();
+ for (int j = 0; j < numDocs & j < values.getLeft().length; j++) {
+ // If current value is not null, we process the data.
+ if (curNull == null || !curNull.contains(j)) {
+ // If existing minimum value is null, we set the value directly.
+ if (nullBitmap != null && nullBitmap.contains(j)) {
+ _intValuesSV[j] = values.getLeft()[j];
+ } else {
+ _intValuesSV[j] = Math.min(_intValuesSV[j], values.getLeft()[j]);
+ }
+ }
+ }
+ if (nullBitmap != null && curNull != null) {
+ nullBitmap.and(curNull);
+ } else {
+ nullBitmap = null;
+ }
+ }
+ return ImmutablePair.of(_intValuesSV, nullBitmap);
+ }
+
@Override
public long[] transformToLongValuesSV(ValueBlock valueBlock) {
int numDocs = valueBlock.getNumDocs();
@@ -59,6 +101,36 @@ public class LeastTransformFunction extends
SelectTupleElementTransformFunction
return _longValuesSV;
}
+ @Override
+ public Pair<long[], RoaringBitmap>
transformToLongValuesSVWithNull(ValueBlock valueBlock) {
+ int numDocs = valueBlock.getNumDocs();
+ initLongValuesSV(numDocs);
+ Pair<long[], RoaringBitmap> values =
_arguments.get(0).transformToLongValuesSVWithNull(valueBlock);
+ System.arraycopy(values.getLeft(), 0, _longValuesSV, 0, numDocs);
+ RoaringBitmap nullBitmap = values.getRight();
+ for (int i = 1; i < _arguments.size(); i++) {
+ values = _arguments.get(i).transformToLongValuesSVWithNull(valueBlock);
+ RoaringBitmap curNull = values.getRight();
+ for (int j = 0; j < numDocs & j < values.getLeft().length; j++) {
+ // If current value is not null, we process the data.
+ if (curNull == null || !curNull.contains(j)) {
+ // If existing minimum value is null, we set the value directly.
+ if (nullBitmap != null && nullBitmap.contains(j)) {
+ _longValuesSV[j] = values.getLeft()[j];
+ } else {
+ _longValuesSV[j] = Math.min(_longValuesSV[j], values.getLeft()[j]);
+ }
+ }
+ }
+ if (nullBitmap != null && curNull != null) {
+ nullBitmap.and(curNull);
+ } else {
+ nullBitmap = null;
+ }
+ }
+ return ImmutablePair.of(_longValuesSV, nullBitmap);
+ }
+
@Override
public float[] transformToFloatValuesSV(ValueBlock valueBlock) {
int numDocs = valueBlock.getNumDocs();
@@ -74,6 +146,36 @@ public class LeastTransformFunction extends
SelectTupleElementTransformFunction
return _floatValuesSV;
}
+ @Override
+ public Pair<float[], RoaringBitmap>
transformToFloatValuesSVWithNull(ValueBlock valueBlock) {
+ int numDocs = valueBlock.getNumDocs();
+ initFloatValuesSV(numDocs);
+ Pair<float[], RoaringBitmap> values =
_arguments.get(0).transformToFloatValuesSVWithNull(valueBlock);
+ System.arraycopy(values.getLeft(), 0, _floatValuesSV, 0, numDocs);
+ RoaringBitmap nullBitmap = values.getRight();
+ for (int i = 1; i < _arguments.size(); i++) {
+ values = _arguments.get(i).transformToFloatValuesSVWithNull(valueBlock);
+ RoaringBitmap curNull = values.getRight();
+ for (int j = 0; j < numDocs & j < values.getLeft().length; j++) {
+ // If current value is not null, we process the data.
+ if (curNull != null || !curNull.contains(j)) {
+ // If existing minimum value is null, we set the value directly.
+ if (nullBitmap != null && nullBitmap.contains(j)) {
+ _floatValuesSV[j] = values.getLeft()[j];
+ } else {
+ _floatValuesSV[j] = Math.min(_floatValuesSV[j],
values.getLeft()[j]);
+ }
+ }
+ }
+ if (nullBitmap != null && curNull != null) {
+ nullBitmap.and(curNull);
+ } else {
+ nullBitmap = null;
+ }
+ }
+ return ImmutablePair.of(_floatValuesSV, nullBitmap);
+ }
+
@Override
public double[] transformToDoubleValuesSV(ValueBlock valueBlock) {
int numDocs = valueBlock.getNumDocs();
@@ -89,6 +191,36 @@ public class LeastTransformFunction extends
SelectTupleElementTransformFunction
return _doubleValuesSV;
}
+ @Override
+ public Pair<double[], RoaringBitmap>
transformToDoubleValuesSVWithNull(ValueBlock valueBlock) {
+ int numDocs = valueBlock.getNumDocs();
+ initDoubleValuesSV(numDocs);
+ Pair<double[], RoaringBitmap> values =
_arguments.get(0).transformToDoubleValuesSVWithNull(valueBlock);
+ System.arraycopy(values.getLeft(), 0, _doubleValuesSV, 0, numDocs);
+ RoaringBitmap nullBitmap = values.getRight();
+ for (int i = 1; i < _arguments.size(); i++) {
+ values = _arguments.get(i).transformToDoubleValuesSVWithNull(valueBlock);
+ RoaringBitmap curNull = values.getRight();
+ for (int j = 0; j < numDocs & j < values.getLeft().length; j++) {
+ // If current value is not null, we process the data.
+ if (curNull == null || !curNull.contains(j)) {
+ // If existing minimum value is null, we set the value directly.
+ if (nullBitmap != null && nullBitmap.contains(j)) {
+ _doubleValuesSV[j] = values.getLeft()[j];
+ } else {
+ _doubleValuesSV[j] = Math.min(_doubleValuesSV[j],
values.getLeft()[j]);
+ }
+ }
+ }
+ if (nullBitmap != null && curNull != null) {
+ nullBitmap.and(curNull);
+ } else {
+ nullBitmap = null;
+ }
+ }
+ return ImmutablePair.of(_doubleValuesSV, nullBitmap);
+ }
+
@Override
public BigDecimal[] transformToBigDecimalValuesSV(ValueBlock valueBlock) {
int numDocs = valueBlock.getNumDocs();
@@ -104,6 +236,36 @@ public class LeastTransformFunction extends
SelectTupleElementTransformFunction
return _bigDecimalValuesSV;
}
+ @Override
+ public Pair<BigDecimal[], RoaringBitmap>
transformToBigDecimalValuesSVWithNull(ValueBlock valueBlock) {
+ int numDocs = valueBlock.getNumDocs();
+ initBigDecimalValuesSV(numDocs);
+ Pair<BigDecimal[], RoaringBitmap> values =
_arguments.get(0).transformToBigDecimalValuesSVWithNull(valueBlock);
+ System.arraycopy(values.getLeft(), 0, _bigDecimalValuesSV, 0, numDocs);
+ RoaringBitmap nullBitmap = values.getRight();
+ for (int i = 1; i < _arguments.size(); i++) {
+ values =
_arguments.get(i).transformToBigDecimalValuesSVWithNull(valueBlock);
+ RoaringBitmap curNull = values.getRight();
+ for (int j = 0; j < numDocs & j < values.getLeft().length; j++) {
+ // If current value is not null, we process the data.
+ if (curNull == null || !curNull.contains(j)) {
+ // If existing minimum value is null, we set the value directly.
+ if (nullBitmap != null && nullBitmap.contains(j)) {
+ _bigDecimalValuesSV[j] = values.getLeft()[j];
+ } else {
+ _bigDecimalValuesSV[j] =
_bigDecimalValuesSV[j].min(values.getLeft()[j]);
+ }
+ }
+ }
+ if (nullBitmap != null && curNull != null) {
+ nullBitmap.and(curNull);
+ } else {
+ nullBitmap = null;
+ }
+ }
+ return ImmutablePair.of(_bigDecimalValuesSV, nullBitmap);
+ }
+
@Override
public String[] transformToStringValuesSV(ValueBlock valueBlock) {
int numDocs = valueBlock.getNumDocs();
@@ -120,4 +282,34 @@ public class LeastTransformFunction extends
SelectTupleElementTransformFunction
}
return _stringValuesSV;
}
+
+ @Override
+ public Pair<String[], RoaringBitmap>
transformToStringValuesSVWithNull(ValueBlock valueBlock) {
+ int numDocs = valueBlock.getNumDocs();
+ initStringValuesSV(numDocs);
+ Pair<String[], RoaringBitmap> values =
_arguments.get(0).transformToStringValuesSVWithNull(valueBlock);
+ System.arraycopy(values.getLeft(), 0, _stringValuesSV, 0, numDocs);
+ RoaringBitmap nullBitmap = values.getRight();
+ for (int i = 1; i < _arguments.size(); i++) {
+ values = _arguments.get(i).transformToStringValuesSVWithNull(valueBlock);
+ RoaringBitmap curNull = values.getRight();
+ for (int j = 0; j < numDocs & j < values.getLeft().length; j++) {
+ // If current value is not null, we process the data.
+ if (curNull == null || !curNull.contains(j)) {
+ // If existing minimum value is null, we set the value directly.
+ if (nullBitmap != null && nullBitmap.contains(j)) {
+ _stringValuesSV[j] = values.getLeft()[j];
+ } else if (_stringValuesSV[j].compareTo(values.getLeft()[j]) > 0) {
+ _stringValuesSV[j] = values.getLeft()[j];
+ }
+ }
+ }
+ if (nullBitmap != null && curNull != null) {
+ nullBitmap.and(curNull);
+ } else {
+ nullBitmap = null;
+ }
+ }
+ return ImmutablePair.of(_stringValuesSV, nullBitmap);
+ }
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/SelectTupleElementTransformFunction.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/SelectTupleElementTransformFunction.java
index a4f29f8e9e..584efef5fd 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/SelectTupleElementTransformFunction.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/SelectTupleElementTransformFunction.java
@@ -23,15 +23,17 @@ import java.util.EnumSet;
import java.util.List;
import java.util.Map;
import org.apache.pinot.core.operator.ColumnContext;
+import org.apache.pinot.core.operator.blocks.ValueBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
import org.apache.pinot.spi.data.FieldSpec;
+import org.roaringbitmap.RoaringBitmap;
public abstract class SelectTupleElementTransformFunction extends
BaseTransformFunction {
private static final EnumSet<FieldSpec.DataType> SUPPORTED_DATATYPES =
EnumSet.of(FieldSpec.DataType.INT,
FieldSpec.DataType.LONG, FieldSpec.DataType.FLOAT,
FieldSpec.DataType.DOUBLE, FieldSpec.DataType.BIG_DECIMAL,
- FieldSpec.DataType.TIMESTAMP, FieldSpec.DataType.STRING);
+ FieldSpec.DataType.TIMESTAMP, FieldSpec.DataType.STRING,
FieldSpec.DataType.UNKNOWN);
private static final EnumMap<FieldSpec.DataType,
EnumSet<FieldSpec.DataType>> ACCEPTABLE_COMBINATIONS =
createAcceptableCombinations();
@@ -63,7 +65,8 @@ public abstract class SelectTupleElementTransformFunction
extends BaseTransformF
}
if (dataType == null) {
dataType = argumentType;
- } else if (ACCEPTABLE_COMBINATIONS.get(dataType).contains(argumentType))
{
+ } else if (dataType.isUnknown() || argumentType.isUnknown() ||
ACCEPTABLE_COMBINATIONS.get(dataType)
+ .contains(argumentType)) {
dataType = getLowestCommonDenominatorType(dataType, argumentType);
} else {
throw new IllegalArgumentException(
@@ -84,6 +87,25 @@ public abstract class SelectTupleElementTransformFunction
extends BaseTransformF
return _name;
}
+ @Override
+ public RoaringBitmap getNullBitmap(ValueBlock valueBlock) {
+ RoaringBitmap bitmap = _arguments.get(0).getNullBitmap(valueBlock);
+ if (bitmap == null || bitmap.isEmpty()) {
+ return bitmap;
+ }
+ for (int i = 1; i < _arguments.size(); i++) {
+ RoaringBitmap curBitmap = _arguments.get(i).getNullBitmap(valueBlock);
+ if (curBitmap == null || curBitmap.isEmpty()) {
+ return curBitmap;
+ }
+ bitmap.and(curBitmap);
+ if (bitmap.isEmpty()) {
+ return null;
+ }
+ }
+ return bitmap;
+ }
+
private static FieldSpec.DataType
getLowestCommonDenominatorType(FieldSpec.DataType left, FieldSpec.DataType
right) {
if (left == null || left == right) {
return right;
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/TupleSelectionTransformFunctionsTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/TupleSelectionTransformFunctionsTest.java
index ac96cbf344..0ad118dbdd 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/TupleSelectionTransformFunctionsTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/TupleSelectionTransformFunctionsTest.java
@@ -19,11 +19,13 @@
package org.apache.pinot.core.operator.transform.function;
import java.math.BigDecimal;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.common.function.TransformFunctionType;
import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.common.request.context.RequestContextUtils;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.exception.BadQueryRequestException;
+import org.roaringbitmap.RoaringBitmap;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
@@ -214,6 +216,65 @@ public class TupleSelectionTransformFunctionsTest extends
BaseTransformFunctionT
}
}
+ @Test
+ public void testLeastTransformFunctionNullLiteral() {
+ TransformFunction transformFunction =
+ testLeastPreconditions(String.format("least(%s, null, %s)",
INT_SV_COLUMN, DOUBLE_SV_COLUMN));
+ assertEquals(transformFunction.getResultMetadata().getDataType(),
FieldSpec.DataType.DOUBLE);
+ Pair<double[], RoaringBitmap> doubleValues =
transformFunction.transformToDoubleValuesSVWithNull(_projectionBlock);
+ for (int i = 0; i < NUM_ROWS; i++) {
+ assertEquals(doubleValues.getLeft()[i], Math.min(_intSVValues[i],
_doubleSVValues[i]));
+ }
+ assertEquals(doubleValues.getRight(), null);
+ assertEquals(transformFunction.getNullBitmap(_projectionBlock), null);
+ }
+
+ @Test
+ public void testLeastTransformFunctionNullColumn() {
+ TransformFunction transformFunction =
+ testLeastPreconditions(String.format("least(%s, null, %s)",
INT_SV_NULL_COLUMN, DOUBLE_SV_COLUMN));
+ assertEquals(transformFunction.getResultMetadata().getDataType(),
FieldSpec.DataType.DOUBLE);
+ Pair<double[], RoaringBitmap> doubleValues =
transformFunction.transformToDoubleValuesSVWithNull(_projectionBlock);
+ for (int i = 0; i < NUM_ROWS; i++) {
+ if (i % 2 == 0) {
+ assertEquals(doubleValues.getLeft()[i], Math.min(_intSVValues[i],
_doubleSVValues[i]));
+ } else {
+ assertEquals(doubleValues.getLeft()[i], _doubleSVValues[i]);
+ }
+ }
+ assertEquals(doubleValues.getRight(), null);
+ assertEquals(transformFunction.getNullBitmap(_projectionBlock), null);
+ }
+
+ @Test
+ public void testLeastTransformFunctionAllNulls() {
+ TransformFunction transformFunction =
testLeastPreconditions(String.format("least(null, null, null)"));
+ assertEquals(transformFunction.getResultMetadata().getDataType(),
FieldSpec.DataType.UNKNOWN);
+ Pair<double[], RoaringBitmap> doubleValues =
transformFunction.transformToDoubleValuesSVWithNull(_projectionBlock);
+ RoaringBitmap expectedNull = new RoaringBitmap();
+ expectedNull.add(0L, NUM_ROWS);
+ assertEquals(doubleValues.getRight(), expectedNull);
+ assertEquals(transformFunction.getNullBitmap(_projectionBlock),
expectedNull);
+ }
+
+ @Test
+ public void testLeastTransformFunctionPartialAllNulls() {
+ TransformFunction transformFunction = testLeastPreconditions(
+ String.format("least(%s, %s, %s)", INT_SV_NULL_COLUMN,
INT_SV_NULL_COLUMN, INT_SV_NULL_COLUMN));
+ assertEquals(transformFunction.getResultMetadata().getDataType(),
FieldSpec.DataType.INT);
+ Pair<int[], RoaringBitmap> intValues =
transformFunction.transformToIntValuesSVWithNull(_projectionBlock);
+ RoaringBitmap expectedNull = new RoaringBitmap();
+ for (int i = 0; i < NUM_ROWS; i++) {
+ if (i % 2 == 0) {
+ assertEquals(intValues.getLeft()[i], _intSVValues[i]);
+ } else {
+ expectedNull.add(i);
+ }
+ }
+ assertEquals(intValues.getRight(), expectedNull);
+ assertEquals(transformFunction.getNullBitmap(_projectionBlock),
expectedNull);
+ }
+
@Test(dataProvider = "rejectedParameters", expectedExceptions =
BadQueryRequestException.class)
public void testRejectLeast(String params) {
testGreatestPreconditions("least" + params);
@@ -303,6 +364,65 @@ public class TupleSelectionTransformFunctionsTest extends
BaseTransformFunctionT
}
}
+ @Test
+ public void testGreatestTransformFunctionNullLiteral() {
+ TransformFunction transformFunction =
+ testGreatestPreconditions(String.format("greatest(%s, null, %s)",
INT_SV_COLUMN, DOUBLE_SV_COLUMN));
+ assertEquals(transformFunction.getResultMetadata().getDataType(),
FieldSpec.DataType.DOUBLE);
+ Pair<double[], RoaringBitmap> doubleValues =
transformFunction.transformToDoubleValuesSVWithNull(_projectionBlock);
+ for (int i = 0; i < NUM_ROWS; i++) {
+ assertEquals(doubleValues.getLeft()[i], Math.max(_intSVValues[i],
_doubleSVValues[i]));
+ }
+ assertEquals(doubleValues.getRight(), null);
+ assertEquals(transformFunction.getNullBitmap(_projectionBlock), null);
+ }
+
+ @Test
+ public void testGreatestTransformFunctionNullColumn() {
+ TransformFunction transformFunction =
+ testGreatestPreconditions(String.format("greatest(%s, null, %s)",
INT_SV_NULL_COLUMN, DOUBLE_SV_COLUMN));
+ assertEquals(transformFunction.getResultMetadata().getDataType(),
FieldSpec.DataType.DOUBLE);
+ Pair<double[], RoaringBitmap> doubleValues =
transformFunction.transformToDoubleValuesSVWithNull(_projectionBlock);
+ for (int i = 0; i < NUM_ROWS; i++) {
+ if (i % 2 == 0) {
+ assertEquals(doubleValues.getLeft()[i], Math.max(_intSVValues[i],
_doubleSVValues[i]));
+ } else {
+ assertEquals(doubleValues.getLeft()[i], _doubleSVValues[i]);
+ }
+ }
+ assertEquals(doubleValues.getRight(), null);
+ assertEquals(transformFunction.getNullBitmap(_projectionBlock), null);
+ }
+
+ @Test
+ public void testGreatestTransformFunctionAllNulls() {
+ TransformFunction transformFunction =
testGreatestPreconditions(String.format("greatest(null, null, null)"));
+ assertEquals(transformFunction.getResultMetadata().getDataType(),
FieldSpec.DataType.UNKNOWN);
+ Pair<double[], RoaringBitmap> doubleValues =
transformFunction.transformToDoubleValuesSVWithNull(_projectionBlock);
+ RoaringBitmap expectedNull = new RoaringBitmap();
+ expectedNull.add(0L, NUM_ROWS);
+ assertEquals(doubleValues.getRight(), expectedNull);
+ assertEquals(transformFunction.getNullBitmap(_projectionBlock),
expectedNull);
+ }
+
+ @Test
+ public void testGreatestTransformFunctionPartialAllNulls() {
+ TransformFunction transformFunction = testGreatestPreconditions(
+ String.format("greatest(%s, %s, %s)", INT_SV_NULL_COLUMN,
INT_SV_NULL_COLUMN, INT_SV_NULL_COLUMN));
+ assertEquals(transformFunction.getResultMetadata().getDataType(),
FieldSpec.DataType.INT);
+ Pair<int[], RoaringBitmap> intValues =
transformFunction.transformToIntValuesSVWithNull(_projectionBlock);
+ RoaringBitmap expectedNull = new RoaringBitmap();
+ for (int i = 0; i < NUM_ROWS; i++) {
+ if (i % 2 == 0) {
+ assertEquals(intValues.getLeft()[i], _intSVValues[i]);
+ } else {
+ expectedNull.add(i);
+ }
+ }
+ assertEquals(intValues.getRight(), expectedNull);
+ assertEquals(transformFunction.getNullBitmap(_projectionBlock),
expectedNull);
+ }
+
@Test
public void testGreatestTransformFunctionString() {
TransformFunction transformFunction = testGreatestPreconditions(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
