>From Ali Alsuliman <[email protected]>:
Ali Alsuliman has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20564?usp=email )
Change subject: parquet null
......................................................................
parquet null
Change-Id: I0774b8003a12909ed4d0a6f4201da0e65c181055
---
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/ParquetConverterContext.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/ObjectRepeatedConverter.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/RepeatedConverter.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/PrimitiveRepeatedConverter.java
4 files changed, 24 insertions(+), 5 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/64/20564/1
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/ParquetConverterContext.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/ParquetConverterContext.java
index db63adc..7e9b3a20 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/ParquetConverterContext.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/ParquetConverterContext.java
@@ -42,6 +42,7 @@
import org.apache.asterix.om.base.AMutableDouble;
import org.apache.asterix.om.base.AMutableInt64;
import org.apache.asterix.om.base.AMutableTime;
+import org.apache.asterix.om.base.ANull;
import org.apache.asterix.om.base.ATime;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.BuiltinType;
@@ -82,6 +83,9 @@
@SuppressWarnings("unchecked")
private final ISerializerDeserializer<ADateTime> datetimeSerDer =
SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ADATETIME);
+ @SuppressWarnings("unchecked")
+ private final ISerializerDeserializer<ANull> nullSerDer =
+
SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ANULL);
//Issued warnings
private final List<Warning> warnings;
@@ -293,6 +297,14 @@
}
}
+ public void serializeNull(DataOutput output) {
+ try {
+ nullSerDer.serialize(ANull.NULL, output);
+ } catch (HyracksDataException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
private void writeLength(int length, DataOutput out) throws IOException {
int requiredLength = VarLenIntEncoderDecoder.getBytesRequired(length);
if (lengthBytes == null || requiredLength > lengthBytes.length) {
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/ObjectRepeatedConverter.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/ObjectRepeatedConverter.java
index ef944ed..1d2e143 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/ObjectRepeatedConverter.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/ObjectRepeatedConverter.java
@@ -56,9 +56,9 @@
*
* Instead of:
* message arrow_schema {
- * required group myGroupArray (LIST) {
+ * required group my_group_list (LIST) {
* repeated group list {
- * optional group {
+ * optional group element {
* optional binary hello (STRING);
* optional binary foo (STRING);
* }
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/RepeatedConverter.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/RepeatedConverter.java
index 87d0ae4..812e358 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/RepeatedConverter.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/RepeatedConverter.java
@@ -31,6 +31,8 @@
import org.apache.parquet.schema.PrimitiveType;
public class RepeatedConverter extends AbstractComplexConverter {
+ private boolean valueAdded;
+
public RepeatedConverter(AbstractComplexConverter parent, int index,
GroupType parquetType,
ParquetConverterContext context) throws IOException {
super(parent, index, parquetType, context);
@@ -38,12 +40,16 @@
@Override
public void start() {
- //NoOp
+ valueAdded = false;
}
@Override
public void end() {
- //NoOp
+ if (!valueAdded) {
+ context.serializeNull(getDataOutput());
+ parent.addValue(this);
+ }
+ valueAdded = false;
}
@Override
@@ -53,6 +59,7 @@
@Override
public void addValue(IFieldValue value) {
+ valueAdded = true;
parent.addValue(value);
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/PrimitiveRepeatedConverter.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/PrimitiveRepeatedConverter.java
index a832484..45f611c 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/PrimitiveRepeatedConverter.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/PrimitiveRepeatedConverter.java
@@ -48,7 +48,7 @@
* message schema {
* required group my_primitive_array (LIST) {
* repeated group list {
- * optional binary (STRING);
+ * optional binary element (STRING);
* }
* }
* }
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20564?usp=email
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings?usp=email
Gerrit-MessageType: newchange
Gerrit-Project: asterixdb
Gerrit-Branch: phoenix
Gerrit-Change-Id: I0774b8003a12909ed4d0a6f4201da0e65c181055
Gerrit-Change-Number: 20564
Gerrit-PatchSet: 1
Gerrit-Owner: Ali Alsuliman <[email protected]>
