This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 4a263ea7d9 DISTINCT COUNT/SUM/AVG NULL support. (#11348)
4a263ea7d9 is described below
commit 4a263ea7d91d5dfbddbf2731433680c5fd7bdb4d
Author: Shen Yu <[email protected]>
AuthorDate: Mon Aug 21 16:40:44 2023 +0000
DISTINCT COUNT/SUM/AVG NULL support. (#11348)
---
.../function/AggregationFunctionFactory.java | 6 +-
.../BaseDistinctAggregateAggregationFunction.java | 319 ++++++++++++++++++---
.../function/DistinctAvgAggregationFunction.java | 4 +-
.../function/DistinctAvgMVAggregationFunction.java | 2 +-
.../function/DistinctCountAggregationFunction.java | 4 +-
.../DistinctCountMVAggregationFunction.java | 2 +-
.../function/DistinctSumAggregationFunction.java | 4 +-
.../function/DistinctSumMVAggregationFunction.java | 2 +-
.../apache/pinot/queries/AllNullQueriesTest.java | 2 +-
.../queries/NullHandlingEnabledQueriesTest.java | 207 +++++++++++++
10 files changed, 494 insertions(+), 58 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/AggregationFunctionFactory.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/AggregationFunctionFactory.java
index aadc9fbd63..a79f01a0f1 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/AggregationFunctionFactory.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/AggregationFunctionFactory.java
@@ -257,7 +257,7 @@ public class AggregationFunctionFactory {
case MINMAXRANGE:
return new MinMaxRangeAggregationFunction(firstArgument);
case DISTINCTCOUNT:
- return new DistinctCountAggregationFunction(firstArgument);
+ return new DistinctCountAggregationFunction(firstArgument,
nullHandlingEnabled);
case DISTINCTCOUNTBITMAP:
return new DistinctCountBitmapAggregationFunction(firstArgument);
case SEGMENTPARTITIONEDDISTINCTCOUNT:
@@ -275,9 +275,9 @@ public class AggregationFunctionFactory {
case DISTINCTCOUNTRAWTHETASKETCH:
return new
DistinctCountRawThetaSketchAggregationFunction(arguments);
case DISTINCTSUM:
- return new DistinctSumAggregationFunction(firstArgument);
+ return new DistinctSumAggregationFunction(firstArgument,
nullHandlingEnabled);
case DISTINCTAVG:
- return new DistinctAvgAggregationFunction(firstArgument);
+ return new DistinctAvgAggregationFunction(firstArgument,
nullHandlingEnabled);
case IDSET:
return new IdSetAggregationFunction(arguments);
case COUNTMV:
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/BaseDistinctAggregateAggregationFunction.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/BaseDistinctAggregateAggregationFunction.java
index e9bc38e92b..49754b11f0 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/BaseDistinctAggregateAggregationFunction.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/BaseDistinctAggregateAggregationFunction.java
@@ -49,11 +49,13 @@ import org.roaringbitmap.RoaringBitmap;
public abstract class BaseDistinctAggregateAggregationFunction<T extends
Comparable>
extends BaseSingleInputAggregationFunction<Set, T> {
private final AggregationFunctionType _functionType;
+ private final boolean _nullHandlingEnabled;
protected BaseDistinctAggregateAggregationFunction(ExpressionContext
expression,
- AggregationFunctionType aggregationFunctionType) {
+ AggregationFunctionType aggregationFunctionType, boolean
nullHandlingEnabled) {
super(expression);
_functionType = aggregationFunctionType;
+ _nullHandlingEnabled = nullHandlingEnabled;
}
@Override
@@ -144,12 +146,25 @@ public abstract class
BaseDistinctAggregateAggregationFunction<T extends Compara
protected void svAggregate(int length, AggregationResultHolder
aggregationResultHolder,
Map<ExpressionContext, BlockValSet> blockValSetMap) {
BlockValSet blockValSet = blockValSetMap.get(_expression);
+ RoaringBitmap nullBitmap = null;
// For dictionary-encoded expression, store dictionary ids into the bitmap
Dictionary dictionary = blockValSet.getDictionary();
if (dictionary != null) {
+ if (_nullHandlingEnabled) {
+ nullBitmap = blockValSet.getNullBitmap();
+ }
int[] dictIds = blockValSet.getDictionaryIdsSV();
- getDictIdBitmap(aggregationResultHolder, dictionary).addN(dictIds, 0,
length);
+ if (nullBitmap != null && !nullBitmap.isEmpty()) {
+ RoaringBitmap dictIdBitmap = getDictIdBitmap(aggregationResultHolder,
dictionary);
+ for (int i = 0; i < length; i++) {
+ if (!nullBitmap.contains(i)) {
+ dictIdBitmap.add(dictIds[i]);
+ }
+ }
+ } else {
+ getDictIdBitmap(aggregationResultHolder, dictionary).addN(dictIds, 0,
length);
+ }
return;
}
@@ -160,44 +175,110 @@ public abstract class
BaseDistinctAggregateAggregationFunction<T extends Compara
case INT:
IntOpenHashSet intSet = (IntOpenHashSet) valueSet;
int[] intValues = blockValSet.getIntValuesSV();
- for (int i = 0; i < length; i++) {
- intSet.add(intValues[i]);
+ if (_nullHandlingEnabled) {
+ nullBitmap = blockValSet.getNullBitmap();
+ }
+ if (nullBitmap != null && !nullBitmap.isEmpty()) {
+ for (int i = 0; i < length; i++) {
+ if (!nullBitmap.contains(i)) {
+ intSet.add(intValues[i]);
+ }
+ }
+ } else {
+ for (int i = 0; i < length; i++) {
+ intSet.add(intValues[i]);
+ }
}
break;
case LONG:
LongOpenHashSet longSet = (LongOpenHashSet) valueSet;
long[] longValues = blockValSet.getLongValuesSV();
- for (int i = 0; i < length; i++) {
- longSet.add(longValues[i]);
+ if (_nullHandlingEnabled) {
+ nullBitmap = blockValSet.getNullBitmap();
+ }
+ if (nullBitmap != null && !nullBitmap.isEmpty()) {
+ for (int i = 0; i < length; i++) {
+ if (!nullBitmap.contains(i)) {
+ longSet.add(longValues[i]);
+ }
+ }
+ } else {
+ for (int i = 0; i < length; i++) {
+ longSet.add(longValues[i]);
+ }
}
break;
case FLOAT:
FloatOpenHashSet floatSet = (FloatOpenHashSet) valueSet;
float[] floatValues = blockValSet.getFloatValuesSV();
- for (int i = 0; i < length; i++) {
- floatSet.add(floatValues[i]);
+ if (_nullHandlingEnabled) {
+ nullBitmap = blockValSet.getNullBitmap();
+ }
+ if (nullBitmap != null && !nullBitmap.isEmpty()) {
+ for (int i = 0; i < length; i++) {
+ if (!nullBitmap.contains(i)) {
+ floatSet.add(floatValues[i]);
+ }
+ }
+ } else {
+ for (int i = 0; i < length; i++) {
+ floatSet.add(floatValues[i]);
+ }
}
break;
case DOUBLE:
DoubleOpenHashSet doubleSet = (DoubleOpenHashSet) valueSet;
double[] doubleValues = blockValSet.getDoubleValuesSV();
- for (int i = 0; i < length; i++) {
- doubleSet.add(doubleValues[i]);
+ if (_nullHandlingEnabled) {
+ nullBitmap = blockValSet.getNullBitmap();
+ }
+ if (nullBitmap != null && !nullBitmap.isEmpty()) {
+ for (int i = 0; i < length; i++) {
+ if (!nullBitmap.contains(i)) {
+ doubleSet.add(doubleValues[i]);
+ }
+ }
+ } else {
+ for (int i = 0; i < length; i++) {
+ doubleSet.add(doubleValues[i]);
+ }
}
break;
case STRING:
ObjectOpenHashSet<String> stringSet = (ObjectOpenHashSet<String>)
valueSet;
String[] stringValues = blockValSet.getStringValuesSV();
//noinspection ManualArrayToCollectionCopy
- for (int i = 0; i < length; i++) {
- stringSet.add(stringValues[i]);
+ if (_nullHandlingEnabled) {
+ nullBitmap = blockValSet.getNullBitmap();
+ }
+ if (nullBitmap != null && !nullBitmap.isEmpty()) {
+ for (int i = 0; i < length; i++) {
+ if (!nullBitmap.contains(i)) {
+ stringSet.add(stringValues[i]);
+ }
+ }
+ } else {
+ for (int i = 0; i < length; i++) {
+ stringSet.add(stringValues[i]);
+ }
}
break;
case BYTES:
ObjectOpenHashSet<ByteArray> bytesSet = (ObjectOpenHashSet<ByteArray>)
valueSet;
byte[][] bytesValues = blockValSet.getBytesValuesSV();
- for (int i = 0; i < length; i++) {
- bytesSet.add(new ByteArray(bytesValues[i]));
+ if (_nullHandlingEnabled) {
+ nullBitmap = blockValSet.getNullBitmap();
+ }
+ if (nullBitmap != null && !nullBitmap.isEmpty()) {
+ for (int i = 0; i < length; i++) {
+ if (!nullBitmap.contains(i)) {
+ bytesSet.add(new ByteArray(bytesValues[i]));
+ }
+ }
+ } else {
+ for (int i = 0; i < length; i++) {
+ bytesSet.add(new ByteArray(bytesValues[i]));
+ }
}
break;
default:
@@ -287,13 +368,25 @@ public abstract class
BaseDistinctAggregateAggregationFunction<T extends Compara
protected void svAggregateGroupBySV(int length, int[] groupKeyArray,
GroupByResultHolder groupByResultHolder,
Map<ExpressionContext, BlockValSet> blockValSetMap) {
BlockValSet blockValSet = blockValSetMap.get(_expression);
+ RoaringBitmap nullBitmap = null;
// For dictionary-encoded expression, store dictionary ids into the bitmap
Dictionary dictionary = blockValSet.getDictionary();
if (dictionary != null) {
+ if (_nullHandlingEnabled) {
+ nullBitmap = blockValSet.getNullBitmap();
+ }
int[] dictIds = blockValSet.getDictionaryIdsSV();
- for (int i = 0; i < length; i++) {
- getDictIdBitmap(groupByResultHolder, groupKeyArray[i],
dictionary).add(dictIds[i]);
+ if (nullBitmap != null && !nullBitmap.isEmpty()) {
+ for (int i = 0; i < length; i++) {
+ if (!nullBitmap.contains(i)) {
+ getDictIdBitmap(groupByResultHolder, groupKeyArray[i],
dictionary).add(dictIds[i]);
+ }
+ }
+ } else {
+ for (int i = 0; i < length; i++) {
+ getDictIdBitmap(groupByResultHolder, groupKeyArray[i],
dictionary).add(dictIds[i]);
+ }
}
return;
}
@@ -303,41 +396,111 @@ public abstract class
BaseDistinctAggregateAggregationFunction<T extends Compara
switch (storedType) {
case INT:
int[] intValues = blockValSet.getIntValuesSV();
- for (int i = 0; i < length; i++) {
- ((IntOpenHashSet) getValueSet(groupByResultHolder, groupKeyArray[i],
DataType.INT)).add(intValues[i]);
+ if (_nullHandlingEnabled) {
+ nullBitmap = blockValSet.getNullBitmap();
+ }
+ if (nullBitmap != null && !nullBitmap.isEmpty()) {
+ for (int i = 0; i < length; i++) {
+ if (!nullBitmap.contains(i)) {
+ ((IntOpenHashSet) getValueSet(groupByResultHolder,
groupKeyArray[i], DataType.INT)).add(intValues[i]);
+ }
+ }
+ } else {
+ for (int i = 0; i < length; i++) {
+ ((IntOpenHashSet) getValueSet(groupByResultHolder,
groupKeyArray[i], DataType.INT)).add(intValues[i]);
+ }
}
break;
case LONG:
long[] longValues = blockValSet.getLongValuesSV();
- for (int i = 0; i < length; i++) {
- ((LongOpenHashSet) getValueSet(groupByResultHolder,
groupKeyArray[i], DataType.LONG)).add(longValues[i]);
+ if (_nullHandlingEnabled) {
+ nullBitmap = blockValSet.getNullBitmap();
+ }
+ if (nullBitmap != null && !nullBitmap.isEmpty()) {
+ for (int i = 0; i < length; i++) {
+ if (!nullBitmap.contains(i)) {
+ ((LongOpenHashSet) getValueSet(groupByResultHolder,
groupKeyArray[i], DataType.LONG)).add(longValues[i]);
+ }
+ }
+ } else {
+ for (int i = 0; i < length; i++) {
+ ((LongOpenHashSet) getValueSet(groupByResultHolder,
groupKeyArray[i], DataType.LONG)).add(longValues[i]);
+ }
}
break;
case FLOAT:
float[] floatValues = blockValSet.getFloatValuesSV();
- for (int i = 0; i < length; i++) {
- ((FloatOpenHashSet) getValueSet(groupByResultHolder,
groupKeyArray[i], DataType.FLOAT)).add(floatValues[i]);
+ if (_nullHandlingEnabled) {
+ nullBitmap = blockValSet.getNullBitmap();
+ }
+ if (nullBitmap != null && !nullBitmap.isEmpty()) {
+ for (int i = 0; i < length; i++) {
+ if (!nullBitmap.contains(i)) {
+ ((FloatOpenHashSet) getValueSet(groupByResultHolder,
groupKeyArray[i], DataType.FLOAT)).add(
+ floatValues[i]);
+ }
+ }
+ } else {
+ for (int i = 0; i < length; i++) {
+ ((FloatOpenHashSet) getValueSet(groupByResultHolder,
groupKeyArray[i], DataType.FLOAT)).add(floatValues[i]);
+ }
}
break;
case DOUBLE:
double[] doubleValues = blockValSet.getDoubleValuesSV();
- for (int i = 0; i < length; i++) {
- ((DoubleOpenHashSet) getValueSet(groupByResultHolder,
groupKeyArray[i], DataType.DOUBLE)).add(
- doubleValues[i]);
+ if (_nullHandlingEnabled) {
+ nullBitmap = blockValSet.getNullBitmap();
+ }
+ if (nullBitmap != null && !nullBitmap.isEmpty()) {
+ for (int i = 0; i < length; i++) {
+ if (!nullBitmap.contains(i)) {
+ ((DoubleOpenHashSet) getValueSet(groupByResultHolder,
groupKeyArray[i], DataType.DOUBLE)).add(
+ doubleValues[i]);
+ }
+ }
+ } else {
+ for (int i = 0; i < length; i++) {
+ ((DoubleOpenHashSet) getValueSet(groupByResultHolder,
groupKeyArray[i], DataType.DOUBLE)).add(
+ doubleValues[i]);
+ }
}
break;
case STRING:
String[] stringValues = blockValSet.getStringValuesSV();
- for (int i = 0; i < length; i++) {
- ((ObjectOpenHashSet<String>) getValueSet(groupByResultHolder,
groupKeyArray[i], DataType.STRING)).add(
- stringValues[i]);
+ if (_nullHandlingEnabled) {
+ nullBitmap = blockValSet.getNullBitmap();
+ }
+ if (nullBitmap != null && !nullBitmap.isEmpty()) {
+ for (int i = 0; i < length; i++) {
+ if (!nullBitmap.contains(i)) {
+ ((ObjectOpenHashSet<String>) getValueSet(groupByResultHolder,
groupKeyArray[i], DataType.STRING)).add(
+ stringValues[i]);
+ }
+ }
+ } else {
+ for (int i = 0; i < length; i++) {
+ ((ObjectOpenHashSet<String>) getValueSet(groupByResultHolder,
groupKeyArray[i], DataType.STRING)).add(
+ stringValues[i]);
+ }
}
break;
case BYTES:
byte[][] bytesValues = blockValSet.getBytesValuesSV();
- for (int i = 0; i < length; i++) {
- ((ObjectOpenHashSet<ByteArray>) getValueSet(groupByResultHolder,
groupKeyArray[i], DataType.BYTES)).add(
- new ByteArray(bytesValues[i]));
+ if (_nullHandlingEnabled) {
+ nullBitmap = blockValSet.getNullBitmap();
+ }
+ if (nullBitmap != null && !nullBitmap.isEmpty()) {
+ for (int i = 0; i < length; i++) {
+ if (!nullBitmap.contains(i)) {
+ ((ObjectOpenHashSet<ByteArray>) getValueSet(groupByResultHolder,
groupKeyArray[i], DataType.BYTES)).add(
+ new ByteArray(bytesValues[i]));
+ }
+ }
+ } else {
+ for (int i = 0; i < length; i++) {
+ ((ObjectOpenHashSet<ByteArray>) getValueSet(groupByResultHolder,
groupKeyArray[i], DataType.BYTES)).add(
+ new ByteArray(bytesValues[i]));
+ }
}
break;
default:
@@ -428,13 +591,25 @@ public abstract class
BaseDistinctAggregateAggregationFunction<T extends Compara
protected void svAggregateGroupByMV(int length, int[][] groupKeysArray,
GroupByResultHolder groupByResultHolder,
Map<ExpressionContext, BlockValSet> blockValSetMap) {
BlockValSet blockValSet = blockValSetMap.get(_expression);
+ RoaringBitmap nullBitmap = null;
// For dictionary-encoded expression, store dictionary ids into the bitmap
Dictionary dictionary = blockValSet.getDictionary();
if (dictionary != null) {
+ if (_nullHandlingEnabled) {
+ nullBitmap = blockValSet.getNullBitmap();
+ }
int[] dictIds = blockValSet.getDictionaryIdsSV();
- for (int i = 0; i < length; i++) {
- setDictIdForGroupKeys(groupByResultHolder, groupKeysArray[i],
dictionary, dictIds[i]);
+ if (nullBitmap != null && !nullBitmap.isEmpty()) {
+ for (int i = 0; i < length; i++) {
+ if (!nullBitmap.contains(i)) {
+ setDictIdForGroupKeys(groupByResultHolder, groupKeysArray[i],
dictionary, dictIds[i]);
+ }
+ }
+ } else {
+ for (int i = 0; i < length; i++) {
+ setDictIdForGroupKeys(groupByResultHolder, groupKeysArray[i],
dictionary, dictIds[i]);
+ }
}
return;
}
@@ -444,38 +619,92 @@ public abstract class
BaseDistinctAggregateAggregationFunction<T extends Compara
switch (storedType) {
case INT:
int[] intValues = blockValSet.getIntValuesSV();
- for (int i = 0; i < length; i++) {
- setValueForGroupKeys(groupByResultHolder, groupKeysArray[i],
intValues[i]);
+ nullBitmap = blockValSet.getNullBitmap();
+ if (nullBitmap != null && !nullBitmap.isEmpty()) {
+ for (int i = 0; i < length; i++) {
+ if (!nullBitmap.contains(i)) {
+ setValueForGroupKeys(groupByResultHolder, groupKeysArray[i],
intValues[i]);
+ }
+ }
+ } else {
+ for (int i = 0; i < length; i++) {
+ setValueForGroupKeys(groupByResultHolder, groupKeysArray[i],
intValues[i]);
+ }
}
break;
case LONG:
long[] longValues = blockValSet.getLongValuesSV();
- for (int i = 0; i < length; i++) {
- setValueForGroupKeys(groupByResultHolder, groupKeysArray[i],
longValues[i]);
+ nullBitmap = blockValSet.getNullBitmap();
+ if (nullBitmap != null && !nullBitmap.isEmpty()) {
+ for (int i = 0; i < length; i++) {
+ if (!nullBitmap.contains(i)) {
+ setValueForGroupKeys(groupByResultHolder, groupKeysArray[i],
longValues[i]);
+ }
+ }
+ } else {
+ for (int i = 0; i < length; i++) {
+ setValueForGroupKeys(groupByResultHolder, groupKeysArray[i],
longValues[i]);
+ }
}
break;
case FLOAT:
float[] floatValues = blockValSet.getFloatValuesSV();
- for (int i = 0; i < length; i++) {
- setValueForGroupKeys(groupByResultHolder, groupKeysArray[i],
floatValues[i]);
+ nullBitmap = blockValSet.getNullBitmap();
+ if (nullBitmap != null && !nullBitmap.isEmpty()) {
+ for (int i = 0; i < length; i++) {
+ if (!nullBitmap.contains(i)) {
+ setValueForGroupKeys(groupByResultHolder, groupKeysArray[i],
floatValues[i]);
+ }
+ }
+ } else {
+ for (int i = 0; i < length; i++) {
+ setValueForGroupKeys(groupByResultHolder, groupKeysArray[i],
floatValues[i]);
+ }
}
break;
case DOUBLE:
double[] doubleValues = blockValSet.getDoubleValuesSV();
- for (int i = 0; i < length; i++) {
- setValueForGroupKeys(groupByResultHolder, groupKeysArray[i],
doubleValues[i]);
+ nullBitmap = blockValSet.getNullBitmap();
+ if (nullBitmap != null && !nullBitmap.isEmpty()) {
+ for (int i = 0; i < length; i++) {
+ if (!nullBitmap.contains(i)) {
+ setValueForGroupKeys(groupByResultHolder, groupKeysArray[i],
doubleValues[i]);
+ }
+ }
+ } else {
+ for (int i = 0; i < length; i++) {
+ setValueForGroupKeys(groupByResultHolder, groupKeysArray[i],
doubleValues[i]);
+ }
}
break;
case STRING:
String[] stringValues = blockValSet.getStringValuesSV();
- for (int i = 0; i < length; i++) {
- setValueForGroupKeys(groupByResultHolder, groupKeysArray[i],
stringValues[i]);
+ nullBitmap = blockValSet.getNullBitmap();
+ if (nullBitmap != null && !nullBitmap.isEmpty()) {
+ for (int i = 0; i < length; i++) {
+ if (!nullBitmap.contains(i)) {
+ setValueForGroupKeys(groupByResultHolder, groupKeysArray[i],
stringValues[i]);
+ }
+ }
+ } else {
+ for (int i = 0; i < length; i++) {
+ setValueForGroupKeys(groupByResultHolder, groupKeysArray[i],
stringValues[i]);
+ }
}
break;
case BYTES:
byte[][] bytesValues = blockValSet.getBytesValuesSV();
- for (int i = 0; i < length; i++) {
- setValueForGroupKeys(groupByResultHolder, groupKeysArray[i], new
ByteArray(bytesValues[i]));
+ nullBitmap = blockValSet.getNullBitmap();
+ if (nullBitmap != null && !nullBitmap.isEmpty()) {
+ for (int i = 0; i < length; i++) {
+ if (!nullBitmap.contains(i)) {
+ setValueForGroupKeys(groupByResultHolder, groupKeysArray[i], new
ByteArray(bytesValues[i]));
+ }
+ }
+ } else {
+ for (int i = 0; i < length; i++) {
+ setValueForGroupKeys(groupByResultHolder, groupKeysArray[i], new
ByteArray(bytesValues[i]));
+ }
}
break;
default:
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctAvgAggregationFunction.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctAvgAggregationFunction.java
index bca958957f..f3d45eed27 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctAvgAggregationFunction.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctAvgAggregationFunction.java
@@ -33,8 +33,8 @@ import org.apache.pinot.segment.spi.AggregationFunctionType;
*/
public class DistinctAvgAggregationFunction extends
BaseDistinctAggregateAggregationFunction<Double> {
- public DistinctAvgAggregationFunction(ExpressionContext expression) {
- super(expression, AggregationFunctionType.DISTINCTAVG);
+ public DistinctAvgAggregationFunction(ExpressionContext expression, boolean
nullHandlingEnabled) {
+ super(expression, AggregationFunctionType.DISTINCTAVG,
nullHandlingEnabled);
}
@Override
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctAvgMVAggregationFunction.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctAvgMVAggregationFunction.java
index 30a8e00492..5f7f04e8ee 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctAvgMVAggregationFunction.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctAvgMVAggregationFunction.java
@@ -34,7 +34,7 @@ import org.apache.pinot.segment.spi.AggregationFunctionType;
public class DistinctAvgMVAggregationFunction extends
BaseDistinctAggregateAggregationFunction<Double> {
public DistinctAvgMVAggregationFunction(ExpressionContext expression) {
- super(expression, AggregationFunctionType.DISTINCTAVGMV);
+ super(expression, AggregationFunctionType.DISTINCTAVGMV, false);
}
@Override
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctCountAggregationFunction.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctCountAggregationFunction.java
index aec983b8df..a31779ab6b 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctCountAggregationFunction.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctCountAggregationFunction.java
@@ -33,8 +33,8 @@ import org.apache.pinot.segment.spi.AggregationFunctionType;
*/
public class DistinctCountAggregationFunction extends
BaseDistinctAggregateAggregationFunction<Integer> {
- public DistinctCountAggregationFunction(ExpressionContext expression) {
- super(expression, AggregationFunctionType.DISTINCTCOUNT);
+ public DistinctCountAggregationFunction(ExpressionContext expression,
boolean nullHandlingEnabled) {
+ super(expression, AggregationFunctionType.DISTINCTCOUNT,
nullHandlingEnabled);
}
@Override
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctCountMVAggregationFunction.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctCountMVAggregationFunction.java
index d8d257b400..cd03a0071f 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctCountMVAggregationFunction.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctCountMVAggregationFunction.java
@@ -34,7 +34,7 @@ import org.apache.pinot.segment.spi.AggregationFunctionType;
public class DistinctCountMVAggregationFunction extends
BaseDistinctAggregateAggregationFunction<Integer> {
public DistinctCountMVAggregationFunction(ExpressionContext expression) {
- super(expression, AggregationFunctionType.DISTINCTCOUNTMV);
+ super(expression, AggregationFunctionType.DISTINCTCOUNTMV, false);
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctSumAggregationFunction.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctSumAggregationFunction.java
index f3a6c55805..c0ee5907dd 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctSumAggregationFunction.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctSumAggregationFunction.java
@@ -33,8 +33,8 @@ import org.apache.pinot.segment.spi.AggregationFunctionType;
*/
public class DistinctSumAggregationFunction extends
BaseDistinctAggregateAggregationFunction<Double> {
- public DistinctSumAggregationFunction(ExpressionContext expression) {
- super(expression, AggregationFunctionType.DISTINCTSUM);
+ public DistinctSumAggregationFunction(ExpressionContext expression, boolean
nullHandlingEnabled) {
+ super(expression, AggregationFunctionType.DISTINCTSUM,
nullHandlingEnabled);
}
@Override
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctSumMVAggregationFunction.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctSumMVAggregationFunction.java
index 3ffd0acd16..78b3d2cf09 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctSumMVAggregationFunction.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctSumMVAggregationFunction.java
@@ -34,7 +34,7 @@ import org.apache.pinot.segment.spi.AggregationFunctionType;
public class DistinctSumMVAggregationFunction extends
BaseDistinctAggregateAggregationFunction<Double> {
public DistinctSumMVAggregationFunction(ExpressionContext expression) {
- super(expression, AggregationFunctionType.DISTINCTSUMMV);
+ super(expression, AggregationFunctionType.DISTINCTSUMMV, false);
}
@Override
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/AllNullQueriesTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/AllNullQueriesTest.java
index f345e99f58..ed8a5a716d 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/AllNullQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/AllNullQueriesTest.java
@@ -560,7 +560,7 @@ public class AllNullQueriesTest extends BaseQueriesTest {
} else if (dataType == DataType.LONG) {
assertEquals(((Double) row[1]).longValue(), Long.MIN_VALUE);
}
- assertEquals(row[2], 1);
+ assertEquals(row[2], 0);
}
}
{
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/NullHandlingEnabledQueriesTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/NullHandlingEnabledQueriesTest.java
index 33eb0b0069..b53bfa17fa 100644
---
a/pinot-core/src/test/java/org/apache/pinot/queries/NullHandlingEnabledQueriesTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/queries/NullHandlingEnabledQueriesTest.java
@@ -479,6 +479,213 @@ public class NullHandlingEnabledQueriesTest extends
BaseQueriesTest {
assertNotNull(resultTable.getRows().get(0)[0]);
}
+ @Test(dataProvider = "NumberTypes")
+ public void testDistinctCountDictNumberTypes(FieldSpec.DataType dataType)
+ throws Exception {
+ initializeRows();
+ insertRow(null);
+ insertRow(1);
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension(COLUMN1, dataType).build();
+ setUpSegments(tableConfig, schema);
+ String query = String.format("SELECT DISTINCTCOUNT(%s) FROM testTable",
COLUMN1);
+
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
QUERY_OPTIONS);
+
+ ResultTable resultTable = brokerResponse.getResultTable();
+ assertEquals(resultTable.getRows().size(), 1);
+ assertEquals(resultTable.getRows().get(0)[0], 1);
+ }
+
+ @Test(dataProvider = "NumberTypes")
+ public void testDistinctCountNonDictNumberTypes(FieldSpec.DataType dataType)
+ throws Exception {
+ initializeRows();
+ insertRow(null);
+ insertRow(1);
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME)
+ .setNoDictionaryColumns(Collections.singletonList(COLUMN1)).build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension(COLUMN1, dataType).build();
+ setUpSegments(tableConfig, schema);
+ String query = String.format("SELECT DISTINCTCOUNT(%s) FROM testTable",
COLUMN1);
+
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
QUERY_OPTIONS);
+
+ ResultTable resultTable = brokerResponse.getResultTable();
+ assertEquals(resultTable.getRows().size(), 1);
+ assertEquals(resultTable.getRows().get(0)[0], 1);
+ }
+
+ @Test(dataProvider = "NumberTypes")
+ public void testGroupByDistinctCountDictNumberTypes(FieldSpec.DataType
dataType)
+ throws Exception {
+ initializeRows();
+ insertRowWithTwoColumns(null, "key");
+ insertRowWithTwoColumns(1, "key");
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension(COLUMN1, dataType)
+ .addSingleValueDimension(COLUMN2, FieldSpec.DataType.STRING).build();
+ setUpSegments(tableConfig, schema);
+ String query = String.format("SELECT DISTINCTCOUNT(%s), %s FROM testTable
GROUP BY %s", COLUMN1, COLUMN2, COLUMN2);
+
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
QUERY_OPTIONS);
+
+ ResultTable resultTable = brokerResponse.getResultTable();
+ assertEquals(resultTable.getRows().size(), 1);
+ assertArrayEquals(resultTable.getRows().get(0), new Object[]{1, "key"});
+ }
+
+ @Test(dataProvider = "NumberTypes")
+ public void testGroupByDistinctCountNonDictNumberTypes(FieldSpec.DataType
dataType)
+ throws Exception {
+ initializeRows();
+ insertRowWithTwoColumns(null, "key");
+ insertRowWithTwoColumns(1, "key");
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME)
+ .setNoDictionaryColumns(Collections.singletonList(COLUMN1)).build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension(COLUMN1, dataType)
+ .addSingleValueDimension(COLUMN2, FieldSpec.DataType.STRING).build();
+ setUpSegments(tableConfig, schema);
+ String query = String.format("SELECT DISTINCTCOUNT(%s), %s FROM testTable
GROUP BY %s", COLUMN1, COLUMN2, COLUMN2);
+
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
QUERY_OPTIONS);
+
+ ResultTable resultTable = brokerResponse.getResultTable();
+ assertEquals(resultTable.getRows().size(), 1);
+ assertArrayEquals(resultTable.getRows().get(0), new Object[]{1, "key"});
+ }
+
+ @Test(dataProvider = "NumberTypes")
+ public void testGroupByMvDistinctCountNonDictNumberTypes(FieldSpec.DataType
dataType)
+ throws Exception {
+ initializeRows();
+ insertRowWithTwoColumns(null, new String[]{"key1", "key2"});
+ insertRowWithTwoColumns(1, new String[]{"key1", "key2"});
+ insertRowWithTwoColumns(2, new String[]{"key2"});
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME)
+ .setNoDictionaryColumns(Collections.singletonList(COLUMN1)).build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension(COLUMN1, dataType)
+ .addMultiValueDimension(COLUMN2, FieldSpec.DataType.STRING).build();
+ setUpSegments(tableConfig, schema);
+ String query =
+ String.format("SELECT DISTINCTCOUNT(%s), %s FROM testTable GROUP BY %s
ORDER BY %s", COLUMN1, COLUMN2, COLUMN2,
+ COLUMN2);
+
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
QUERY_OPTIONS);
+
+ ResultTable resultTable = brokerResponse.getResultTable();
+ assertEquals(resultTable.getRows().get(0), new Object[]{1, "key1"});
+ assertEquals(resultTable.getRows().get(1), new Object[]{2, "key2"});
+ }
+
+ @Test(dataProvider = "NumberTypes")
+ public void testGroupByMvDistinctCountDictNumberTypes(FieldSpec.DataType
dataType)
+ throws Exception {
+ initializeRows();
+ insertRowWithTwoColumns(null, new String[]{"key1", "key2"});
+ insertRowWithTwoColumns(1, new String[]{"key1", "key2"});
+ insertRowWithTwoColumns(2, new String[]{"key2"});
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension(COLUMN1, dataType)
+ .addMultiValueDimension(COLUMN2, FieldSpec.DataType.STRING).build();
+ setUpSegments(tableConfig, schema);
+ String query =
+ String.format("SELECT DISTINCTCOUNT(%s), %s FROM testTable GROUP BY %s
ORDER BY %s", COLUMN1, COLUMN2, COLUMN2,
+ COLUMN2);
+
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
QUERY_OPTIONS);
+
+ ResultTable resultTable = brokerResponse.getResultTable();
+ assertEquals(resultTable.getRows().get(0), new Object[]{1, "key1"});
+ assertEquals(resultTable.getRows().get(1), new Object[]{2, "key2"});
+ }
+
+ @DataProvider(name = "DistinctCountObjectTypes")
+ public static Object[][] getDistinctAggregationObjectTypes() {
+ return new Object[][]{
+ {FieldSpec.DataType.STRING, "a"}, {
+ FieldSpec.DataType.BYTES, "a string".getBytes()
+ }
+ };
+ }
+
+ @Test(dataProvider = "DistinctCountObjectTypes")
+ public void testObjectDistinctCountObjectTypes(FieldSpec.DataType dataType,
Object value)
+ throws Exception {
+ initializeRows();
+ insertRow(null);
+ insertRow(value);
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension(COLUMN1, dataType).build();
+ setUpSegments(tableConfig, schema);
+ String query = String.format("SELECT DISTINCTCOUNT(%s) FROM testTable",
COLUMN1);
+
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
QUERY_OPTIONS);
+
+ ResultTable resultTable = brokerResponse.getResultTable();
+ assertEquals(resultTable.getRows().size(), 1);
+ assertEquals(resultTable.getRows().get(0)[0], 1);
+ }
+
+ @Test(dataProvider = "DistinctCountObjectTypes")
+ public void testGroupByDistinctCountObjectTypes(FieldSpec.DataType dataType,
Object value)
+ throws Exception {
+ initializeRows();
+ insertRowWithTwoColumns(null, "key");
+ insertRowWithTwoColumns(value, "key");
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension(COLUMN1, dataType)
+ .addSingleValueDimension(COLUMN2, FieldSpec.DataType.STRING).build();
+ setUpSegments(tableConfig, schema);
+ String query = String.format("SELECT DISTINCTCOUNT(%s), %s FROM testTable
GROUP BY %s", COLUMN1, COLUMN2, COLUMN2);
+
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
QUERY_OPTIONS);
+
+ ResultTable resultTable = brokerResponse.getResultTable();
+ assertEquals(resultTable.getRows().size(), 1);
+ assertArrayEquals(resultTable.getRows().get(0), new Object[]{1, "key"});
+ }
+
+ @Test
+ public void testDistinctSum()
+ throws Exception {
+ initializeRows();
+ insertRow(null);
+ insertRow(1);
+ insertRow(2);
+ insertRow(2);
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension(COLUMN1,
FieldSpec.DataType.INT).build();
+ setUpSegments(tableConfig, schema);
+ String query = String.format("SELECT DISTINCTSUM(%s) FROM testTable",
COLUMN1);
+
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
QUERY_OPTIONS);
+
+ ResultTable resultTable = brokerResponse.getResultTable();
+ assertEquals(resultTable.getRows().size(), 1);
+ assertEquals(resultTable.getRows().get(0)[0], (double) 3);
+ }
+
+ @Test
+ public void testDistinctAvg()
+ throws Exception {
+ initializeRows();
+ insertRow(null);
+ insertRow(1);
+ insertRow(2);
+ insertRow(2);
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension(COLUMN1,
FieldSpec.DataType.INT).build();
+ setUpSegments(tableConfig, schema);
+ String query = String.format("SELECT DISTINCTAVG(%s) FROM testTable",
COLUMN1);
+
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
QUERY_OPTIONS);
+
+ ResultTable resultTable = brokerResponse.getResultTable();
+ assertEquals(resultTable.getRows().size(), 1);
+ assertEquals(resultTable.getRows().get(0)[0], 1.5);
+ }
+
@Test
public void testTransformBlockValSetGetNullBitmap()
throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]