maheshk114 commented on a change in pull request #2479:
URL: https://github.com/apache/hive/pull/2479#discussion_r686623069
##########
File path:
ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedListColumnReader.java
##########
@@ -129,21 +131,16 @@ private boolean
fetchNextValue(PrimitiveObjectInspector.PrimitiveCategory catego
private void addElement(ListColumnVector lcv, List<Object> elements,
PrimitiveObjectInspector.PrimitiveCategory category, int index) throws
IOException {
lcv.offsets[index] = elements.size();
- // Return directly if last value is null
- if (definitionLevel < maxDefLevel) {
- lcv.isNull[index] = true;
- lcv.lengths[index] = 0;
- // fetch the data from parquet data page for next call
- fetchNextValue(category);
- return;
- }
-
do {
// add all data for an element in ListColumnVector, get out the loop if
there is no data or the data is for new element
+ if (definitionLevel < maxDefLevel) {
+ lcv.lengths[index] = 0;
+ lcv.isNull[index] = true;
+ lcv.noNulls = false;
+ }
elements.add(lastValue);
} while (fetchNextValue(category) && (repetitionLevel != 0));
- lcv.isNull[index] = false;
lcv.lengths[index] = elements.size() - lcv.offsets[index];
Review comment:
lcv.lengths[index] is over written ..in the loop for some condition its
set to 0
##########
File path:
ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedListColumnReader.java
##########
@@ -129,21 +131,16 @@ private boolean
fetchNextValue(PrimitiveObjectInspector.PrimitiveCategory catego
private void addElement(ListColumnVector lcv, List<Object> elements,
PrimitiveObjectInspector.PrimitiveCategory category, int index) throws
IOException {
lcv.offsets[index] = elements.size();
- // Return directly if last value is null
- if (definitionLevel < maxDefLevel) {
- lcv.isNull[index] = true;
- lcv.lengths[index] = 0;
- // fetch the data from parquet data page for next call
- fetchNextValue(category);
- return;
- }
-
do {
// add all data for an element in ListColumnVector, get out the loop if
there is no data or the data is for new element
+ if (definitionLevel < maxDefLevel) {
+ lcv.lengths[index] = 0;
+ lcv.isNull[index] = true;
Review comment:
why this has to be done in a loop ?
##########
File path:
ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedListColumnReader.java
##########
@@ -193,59 +190,59 @@ private List
decodeDictionaryIds(PrimitiveObjectInspector.PrimitiveCategory cate
case SHORT:
resultList = new ArrayList<Integer>(total);
for (int i = 0; i < total; ++i) {
- resultList.add(dictionary.readInteger(intList.get(i)));
+ resultList.add(intList.get(i) == null ? null :
dictionary.readInteger(intList.get(i)));
}
break;
case DATE:
case INTERVAL_YEAR_MONTH:
case LONG:
resultList = new ArrayList<Long>(total);
for (int i = 0; i < total; ++i) {
- resultList.add(dictionary.readLong(intList.get(i)));
+ resultList.add(intList.get(i) == null ? null :
dictionary.readLong(intList.get(i)));
}
break;
case BOOLEAN:
resultList = new ArrayList<Long>(total);
for (int i = 0; i < total; ++i) {
- resultList.add(dictionary.readBoolean(intList.get(i)) ? 1 : 0);
+ resultList.add(intList.get(i) == null ? null :
dictionary.readBoolean(intList.get(i)));
Review comment:
instead of 0 or 1 ..value returned by readBoolean is used
##########
File path:
ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedListColumnReader.java
##########
@@ -129,21 +131,16 @@ private boolean
fetchNextValue(PrimitiveObjectInspector.PrimitiveCategory catego
private void addElement(ListColumnVector lcv, List<Object> elements,
PrimitiveObjectInspector.PrimitiveCategory category, int index) throws
IOException {
lcv.offsets[index] = elements.size();
- // Return directly if last value is null
- if (definitionLevel < maxDefLevel) {
- lcv.isNull[index] = true;
- lcv.lengths[index] = 0;
- // fetch the data from parquet data page for next call
- fetchNextValue(category);
- return;
- }
-
do {
// add all data for an element in ListColumnVector, get out the loop if
there is no data or the data is for new element
+ if (definitionLevel < maxDefLevel) {
Review comment:
in fetchNextvalue ..if (definitionLevel != maxDefLevel) { ..then its
considered null..but in this function its >
##########
File path:
ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedListColumnReader.java
##########
@@ -479,6 +501,9 @@ private boolean compareBytesColumnVector(BytesColumnVector
cv1, BytesColumnVecto
int length2 = cv2.vector.length;
if (length1 == length2) {
for (int i = 0; i < length1; i++) {
+ if (cv1.vector[i] == null && cv2.vector[i] == null) {
+ continue;
Review comment:
why not check the length first and return false.
##########
File path:
ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedListColumnReader.java
##########
@@ -129,21 +131,16 @@ private boolean
fetchNextValue(PrimitiveObjectInspector.PrimitiveCategory catego
private void addElement(ListColumnVector lcv, List<Object> elements,
PrimitiveObjectInspector.PrimitiveCategory category, int index) throws
IOException {
lcv.offsets[index] = elements.size();
- // Return directly if last value is null
Review comment:
i think make it simple ..addElement should add element .. fetchNextValue
can be done by the caller in the loop.
##########
File path: ql/src/test/queries/clientpositive/parquet_map_null_vectorization.q
##########
@@ -0,0 +1,20 @@
+set hive.mapred.mode=nonstrict;
+set hive.vectorized.execution.enabled=true;
+set hive.fetch.task.conversion=none;
+
+DROP TABLE parquet_map_type;
+
+
+CREATE TABLE parquet_map_type (
+id int,
+stringMap map<string, string>
Review comment:
create a table with all datatypes and check for null. You may refer
update_all_types.q.
##########
File path:
ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedListColumnReader.java
##########
@@ -323,7 +341,11 @@ private void
fillColumnVector(PrimitiveObjectInspector.PrimitiveCategory categor
int scale = logicalType.getScale();
lcv.child = new DecimalColumnVector(total, precision, scale);
for (int i = 0; i < valueList.size(); i++) {
- ((DecimalColumnVector) lcv.child).vector[i].set(((List<byte[]>)
valueList).get(i), scale);
+ if (valueList.get(i) == null) {
+ lcv.child.isNull[i] = true;
Review comment:
now that value is not set to null ..compareDecimalColumnVector has to be
modified to check fo isNull ..same for other types also
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
