This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 4125daa4c7 Handle nulls in all SELECT paths (#9169)
4125daa4c7 is described below
commit 4125daa4c7979cb785d9f401cc4142b3dcb6a42b
Author: nizarhejazi <[email protected]>
AuthorDate: Thu Aug 4 20:54:02 2022 -0700
Handle nulls in all SELECT paths (#9169)
---
.../reduce/SelectionOnlyStreamingReducer.java | 36 +++++++++++++++++----
.../query/selection/SelectionOperatorUtils.java | 37 +++++++++++-----------
.../apache/pinot/queries/AllNullQueriesTest.java | 13 ++++++++
3 files changed, 61 insertions(+), 25 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/SelectionOnlyStreamingReducer.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/SelectionOnlyStreamingReducer.java
index 202aa4de73..d040e4a998 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/SelectionOnlyStreamingReducer.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/SelectionOnlyStreamingReducer.java
@@ -28,6 +28,7 @@ import org.apache.pinot.common.utils.DataTable;
import org.apache.pinot.core.query.request.context.QueryContext;
import org.apache.pinot.core.query.selection.SelectionOperatorUtils;
import org.apache.pinot.core.transport.ServerRoutingInstance;
+import org.roaringbitmap.RoaringBitmap;
public class SelectionOnlyStreamingReducer implements StreamingReducer {
@@ -50,16 +51,37 @@ public class SelectionOnlyStreamingReducer implements
StreamingReducer {
// get dataSchema
_dataSchema = _dataSchema == null ? dataTable.getDataSchema() :
_dataSchema;
// TODO: For data table map with more than one data tables, remove
conflicting data tables
- reduceWithoutOrdering(dataTable, _queryContext.getLimit());
+ reduceWithoutOrdering(dataTable, _queryContext.getLimit(),
_queryContext.isNullHandlingEnabled());
}
- private void reduceWithoutOrdering(DataTable dataTable, int limit) {
+ private void reduceWithoutOrdering(DataTable dataTable, int limit, boolean
nullHandlingEnabled) {
+ int numColumns = dataTable.getDataSchema().size();
int numRows = dataTable.getNumberOfRows();
- for (int rowId = 0; rowId < numRows; rowId++) {
- if (_rows.size() < limit) {
- _rows.add(SelectionOperatorUtils.extractRowFromDataTable(dataTable,
rowId));
- } else {
- break;
+ if (nullHandlingEnabled) {
+ RoaringBitmap[] nullBitmaps = new RoaringBitmap[numColumns];;
+ for (int coldId = 0; coldId < numColumns; coldId++) {
+ nullBitmaps[coldId] = dataTable.getNullRowIds(coldId);
+ }
+ for (int rowId = 0; rowId < numRows; rowId++) {
+ if (_rows.size() < limit) {
+ Object[] row =
SelectionOperatorUtils.extractRowFromDataTable(dataTable, rowId);
+ for (int colId = 0; colId < numColumns; colId++) {
+ if (nullBitmaps[colId] != null &&
nullBitmaps[colId].contains(rowId)) {
+ row[colId] = null;
+ }
+ }
+ _rows.add(row);
+ } else {
+ break;
+ }
+ }
+ } else {
+ for (int rowId = 0; rowId < numRows; rowId++) {
+ if (_rows.size() < limit) {
+ _rows.add(SelectionOperatorUtils.extractRowFromDataTable(dataTable,
rowId));
+ } else {
+ break;
+ }
}
}
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/selection/SelectionOperatorUtils.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/selection/SelectionOperatorUtils.java
index 3db1666531..ac0f8d5b91 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/selection/SelectionOperatorUtils.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/selection/SelectionOperatorUtils.java
@@ -431,30 +431,31 @@ public class SelectionOperatorUtils {
List<Object[]> rows = new ArrayList<>(Math.min(limit,
SelectionOperatorUtils.MAX_ROW_HOLDER_INITIAL_CAPACITY));
for (DataTable dataTable : dataTables) {
int numColumns = dataTable.getDataSchema().size();
- RoaringBitmap[] nullBitmaps = null;
+ int numRows = dataTable.getNumberOfRows();
if (nullHandlingEnabled) {
- nullBitmaps = new RoaringBitmap[numColumns];;
+ RoaringBitmap[] nullBitmaps = new RoaringBitmap[numColumns];;
for (int coldId = 0; coldId < numColumns; coldId++) {
nullBitmaps[coldId] = dataTable.getNullRowIds(coldId);
}
- }
-
- int numRows = dataTable.getNumberOfRows();
- for (int rowId = 0; rowId < numRows; rowId++) {
- if (rows.size() < limit) {
- rows.add(extractRowFromDataTable(dataTable, rowId));
- } else {
- break;
- }
- }
-
- if (nullHandlingEnabled) {
for (int rowId = 0; rowId < numRows; rowId++) {
- Object[] row = rows.get(rowId);
- for (int colId = 0; colId < numColumns; colId++) {
- if (nullBitmaps[colId] != null &&
nullBitmaps[colId].contains(rowId)) {
- row[colId] = null;
+ if (rows.size() < limit) {
+ Object[] row = extractRowFromDataTable(dataTable, rowId);
+ for (int colId = 0; colId < numColumns; colId++) {
+ if (nullBitmaps[colId] != null &&
nullBitmaps[colId].contains(rowId)) {
+ row[colId] = null;
+ }
}
+ rows.add(row);
+ } else {
+ break;
+ }
+ }
+ } else {
+ for (int rowId = 0; rowId < numRows; rowId++) {
+ if (rows.size() < limit) {
+ rows.add(extractRowFromDataTable(dataTable, rowId));
+ } else {
+ break;
}
}
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/AllNullQueriesTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/AllNullQueriesTest.java
index c3e957b382..305ed783a5 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/AllNullQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/AllNullQueriesTest.java
@@ -288,6 +288,19 @@ public class AllNullQueriesTest extends BaseQueriesTest {
Map<String, String> queryOptions = new HashMap<>();
queryOptions.put("enableNullHandling", "true");
DataType dataType = columnDataType.toDataType();
+ {
+ String query = String.format("SELECT %s FROM testTable WHERE %s is null
limit 5000", COLUMN_NAME, COLUMN_NAME);
+ BrokerResponseNative brokerResponse = getBrokerResponse(query,
queryOptions);
+ ResultTable resultTable = brokerResponse.getResultTable();
+ DataSchema dataSchema = resultTable.getDataSchema();
+ assertEquals(dataSchema,
+ new DataSchema(new String[]{COLUMN_NAME}, new
ColumnDataType[]{columnDataType}));
+ List<Object[]> rows = resultTable.getRows();
+ assertEquals(rows.size(), 4000);
+ for (Object[] row : rows) {
+ assertNull(row[0]);
+ }
+ }
if (columnDataType != ColumnDataType.STRING) {
{
String query = String.format(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
