This is an automated email from the ASF dual-hosted git repository.
junhao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new cbc789ede2 [Arrow] Throws exception out if accept a null value with
non-null field in ArrowFieldWriter. (#5522)
cbc789ede2 is described below
commit cbc789ede2f0386bbfdfb45a85616b368eb91bfc
Author: xiaolan <[email protected]>
AuthorDate: Thu Apr 24 11:34:00 2025 +0800
[Arrow] Throws exception out if accept a null value with non-null field in
ArrowFieldWriter. (#5522)
---
.../java/org/apache/paimon/arrow/ArrowUtils.java | 2 +-
.../paimon/arrow/vector/ArrowFormatWriter.java | 2 +-
.../vector/OneElementFieldVectorGenerator.java | 2 +-
.../paimon/arrow/writer/ArrowFieldWriter.java | 12 +++-
.../arrow/writer/ArrowFieldWriterFactory.java | 2 +-
.../writer/ArrowFieldWriterFactoryVisitor.java | 35 ++++++-----
.../paimon/arrow/writer/ArrowFieldWriters.java | 73 ++++++++++++----------
7 files changed, 77 insertions(+), 51 deletions(-)
diff --git a/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowUtils.java
b/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowUtils.java
index 0f6a98b7a2..617707697b 100644
--- a/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowUtils.java
+++ b/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowUtils.java
@@ -198,7 +198,7 @@ public class ArrowUtils {
fieldWriters[i] =
rowType.getTypeAt(i)
.accept(ArrowFieldWriterFactoryVisitor.INSTANCE)
- .create(vectors.get(i));
+ .create(vectors.get(i), rowType.isNullable());
}
return fieldWriters;
diff --git
a/paimon-arrow/src/main/java/org/apache/paimon/arrow/vector/ArrowFormatWriter.java
b/paimon-arrow/src/main/java/org/apache/paimon/arrow/vector/ArrowFormatWriter.java
index a377dd9f04..5bd19cfe42 100644
---
a/paimon-arrow/src/main/java/org/apache/paimon/arrow/vector/ArrowFormatWriter.java
+++
b/paimon-arrow/src/main/java/org/apache/paimon/arrow/vector/ArrowFormatWriter.java
@@ -62,7 +62,7 @@ public class ArrowFormatWriter implements AutoCloseable {
.get(i)
.type()
.accept(ArrowFieldWriterFactoryVisitor.INSTANCE)
- .create(vectorSchemaRoot.getVector(i));
+ .create(vectorSchemaRoot.getVector(i),
rowType.isNullable());
}
this.batchSize = writeBatchSize;
diff --git
a/paimon-arrow/src/main/java/org/apache/paimon/arrow/vector/OneElementFieldVectorGenerator.java
b/paimon-arrow/src/main/java/org/apache/paimon/arrow/vector/OneElementFieldVectorGenerator.java
index fad33bf632..d392d903d6 100644
---
a/paimon-arrow/src/main/java/org/apache/paimon/arrow/vector/OneElementFieldVectorGenerator.java
+++
b/paimon-arrow/src/main/java/org/apache/paimon/arrow/vector/OneElementFieldVectorGenerator.java
@@ -44,7 +44,7 @@ public class OneElementFieldVectorGenerator implements
AutoCloseable {
dataField
.type()
.accept(ArrowFieldWriterFactoryVisitor.INSTANCE)
- .create(fieldVector);
+ .create(fieldVector, dataField.type().isNullable());
this.row = new GenericRow(1);
this.row.setField(0, value);
}
diff --git
a/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriter.java
b/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriter.java
index 99a78e863c..77c66cc2b7 100644
---
a/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriter.java
+++
b/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriter.java
@@ -25,14 +25,18 @@ import org.apache.arrow.vector.FieldVector;
import javax.annotation.Nullable;
+import static java.lang.String.format;
+
/** A reusable writer to convert a field into Arrow {@link FieldVector}. */
public abstract class ArrowFieldWriter {
// reusable
protected final FieldVector fieldVector;
+ protected final boolean isNullable;
- public ArrowFieldWriter(FieldVector fieldVector) {
+ public ArrowFieldWriter(FieldVector fieldVector, boolean isNullable) {
this.fieldVector = fieldVector;
+ this.isNullable = isNullable;
}
/** Reset the state of the writer to write the next batch of fields. */
@@ -67,6 +71,12 @@ public abstract class ArrowFieldWriter {
/** Get the value from the row at the given position and write to
specified row index. */
public void write(int rowIndex, DataGetters getters, int pos) {
if (getters.isNullAt(pos)) {
+ if (!isNullable) {
+ throw new IllegalArgumentException(
+ format(
+ "Arrow does not support null values in
non-nullable fields. Field name : %s expected not null but found null",
+ fieldVector.getName()));
+ }
fieldVector.setNull(rowIndex);
} else {
doWrite(rowIndex, getters, pos);
diff --git
a/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriterFactory.java
b/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriterFactory.java
index 651a8d8590..97f5503d73 100644
---
a/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriterFactory.java
+++
b/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriterFactory.java
@@ -24,5 +24,5 @@ import org.apache.arrow.vector.FieldVector;
@FunctionalInterface
public interface ArrowFieldWriterFactory {
- ArrowFieldWriter create(FieldVector fieldVector);
+ ArrowFieldWriter create(FieldVector fieldVector, boolean isNullable);
}
diff --git
a/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriterFactoryVisitor.java
b/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriterFactoryVisitor.java
index 20db359754..3d821bf76c 100644
---
a/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriterFactoryVisitor.java
+++
b/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriterFactoryVisitor.java
@@ -80,9 +80,12 @@ public class ArrowFieldWriterFactoryVisitor implements
DataTypeVisitor<ArrowFiel
@Override
public ArrowFieldWriterFactory visit(DecimalType decimalType) {
- return fieldVector ->
+ return (fieldVector, isNullable) ->
new ArrowFieldWriters.DecimalWriter(
- fieldVector, decimalType.getPrecision(),
decimalType.getScale());
+ fieldVector,
+ decimalType.getPrecision(),
+ decimalType.getScale(),
+ isNullable);
}
@Override
@@ -127,16 +130,16 @@ public class ArrowFieldWriterFactoryVisitor implements
DataTypeVisitor<ArrowFiel
@Override
public ArrowFieldWriterFactory visit(TimestampType timestampType) {
- return fieldVector ->
+ return (fieldVector, isNullable) ->
new ArrowFieldWriters.TimestampWriter(
- fieldVector, timestampType.getPrecision(), null);
+ fieldVector, timestampType.getPrecision(), null,
isNullable);
}
@Override
public ArrowFieldWriterFactory visit(LocalZonedTimestampType
localZonedTimestampType) {
- return fieldVector ->
+ return (fieldVector, isNullable) ->
new ArrowFieldWriters.TimestampWriter(
- fieldVector, localZonedTimestampType.getPrecision(),
null);
+ fieldVector, localZonedTimestampType.getPrecision(),
null, isNullable);
}
@Override
@@ -147,10 +150,12 @@ public class ArrowFieldWriterFactoryVisitor implements
DataTypeVisitor<ArrowFiel
@Override
public ArrowFieldWriterFactory visit(ArrayType arrayType) {
ArrowFieldWriterFactory elementWriterFactory =
arrayType.getElementType().accept(this);
- return fieldVector ->
+ return (fieldVector, isNullable) ->
new ArrowFieldWriters.ArrayWriter(
fieldVector,
- elementWriterFactory.create(((ListVector)
fieldVector).getDataVector()));
+ elementWriterFactory.create(
+ ((ListVector) fieldVector).getDataVector(),
isNullable),
+ isNullable);
}
@Override
@@ -162,26 +167,28 @@ public class ArrowFieldWriterFactoryVisitor implements
DataTypeVisitor<ArrowFiel
public ArrowFieldWriterFactory visit(MapType mapType) {
ArrowFieldWriterFactory keyWriterFactory =
mapType.getKeyType().accept(this);
ArrowFieldWriterFactory valueWriterFactory =
mapType.getValueType().accept(this);
- return fieldVector -> {
+ return (fieldVector, isNullable) -> {
MapVector mapVector = (MapVector) fieldVector;
mapVector.reAlloc();
List<FieldVector> keyValueVectors =
mapVector.getDataVector().getChildrenFromFields();
return new ArrowFieldWriters.MapWriter(
fieldVector,
- keyWriterFactory.create(keyValueVectors.get(0)),
- valueWriterFactory.create(keyValueVectors.get(1)));
+ keyWriterFactory.create(keyValueVectors.get(0),
isNullable),
+ valueWriterFactory.create(keyValueVectors.get(1),
isNullable),
+ isNullable);
};
}
@Override
public ArrowFieldWriterFactory visit(RowType rowType) {
- return fieldVector -> {
+ return (fieldVector, isNullable) -> {
List<FieldVector> children = fieldVector.getChildrenFromFields();
ArrowFieldWriter[] fieldWriters = new
ArrowFieldWriter[children.size()];
for (int i = 0; i < children.size(); i++) {
- fieldWriters[i] =
rowType.getTypeAt(i).accept(this).create(children.get(i));
+ fieldWriters[i] =
+
rowType.getTypeAt(i).accept(this).create(children.get(i), isNullable);
}
- return new ArrowFieldWriters.RowWriter(fieldVector, fieldWriters);
+ return new ArrowFieldWriters.RowWriter(fieldVector, fieldWriters,
isNullable);
};
}
}
diff --git
a/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriters.java
b/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriters.java
index 053f93e1ac..feb3bc2460 100644
---
a/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriters.java
+++
b/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriters.java
@@ -70,8 +70,8 @@ public class ArrowFieldWriters {
/** Writer for CHAR & VARCHAR. */
public static class StringWriter extends ArrowFieldWriter {
- public StringWriter(FieldVector fieldVector) {
- super(fieldVector);
+ public StringWriter(FieldVector fieldVector, boolean isNullable) {
+ super(fieldVector, isNullable);
}
@Override
@@ -101,8 +101,8 @@ public class ArrowFieldWriters {
/** Writer for BOOLEAN. */
public static class BooleanWriter extends ArrowFieldWriter {
- public BooleanWriter(FieldVector fieldVector) {
- super(fieldVector);
+ public BooleanWriter(FieldVector fieldVector, boolean isNullable) {
+ super(fieldVector, isNullable);
}
@Override
@@ -132,8 +132,8 @@ public class ArrowFieldWriters {
/** Writer for BINARY & VARBINARY. */
public static class BinaryWriter extends ArrowFieldWriter {
- public BinaryWriter(FieldVector fieldVector) {
- super(fieldVector);
+ public BinaryWriter(FieldVector fieldVector, boolean isNullable) {
+ super(fieldVector, isNullable);
}
@Override
@@ -166,8 +166,9 @@ public class ArrowFieldWriters {
private final int precision;
private final int scale;
- public DecimalWriter(FieldVector fieldVector, int precision, int
scale) {
- super(fieldVector);
+ public DecimalWriter(
+ FieldVector fieldVector, int precision, int scale, boolean
isNullable) {
+ super(fieldVector, isNullable);
this.precision = precision;
this.scale = scale;
}
@@ -203,8 +204,8 @@ public class ArrowFieldWriters {
/** Writer for TINYINT. */
public static class TinyIntWriter extends ArrowFieldWriter {
- public TinyIntWriter(FieldVector fieldVector) {
- super(fieldVector);
+ public TinyIntWriter(FieldVector fieldVector, boolean isNullable) {
+ super(fieldVector, isNullable);
}
@Override
@@ -234,8 +235,8 @@ public class ArrowFieldWriters {
/** Writer for SMALLINT. */
public static class SmallIntWriter extends ArrowFieldWriter {
- public SmallIntWriter(FieldVector fieldVector) {
- super(fieldVector);
+ public SmallIntWriter(FieldVector fieldVector, boolean isNullable) {
+ super(fieldVector, isNullable);
}
@Override
@@ -265,8 +266,8 @@ public class ArrowFieldWriters {
/** Writer for INT. */
public static class IntWriter extends ArrowFieldWriter {
- public IntWriter(FieldVector fieldVector) {
- super(fieldVector);
+ public IntWriter(FieldVector fieldVector, boolean isNullable) {
+ super(fieldVector, isNullable);
}
@Override
@@ -296,8 +297,8 @@ public class ArrowFieldWriters {
/** Writer for BIGINT. */
public static class BigIntWriter extends ArrowFieldWriter {
- public BigIntWriter(FieldVector fieldVector) {
- super(fieldVector);
+ public BigIntWriter(FieldVector fieldVector, boolean isNullable) {
+ super(fieldVector, isNullable);
}
@Override
@@ -324,8 +325,8 @@ public class ArrowFieldWriters {
/** Writer for FLOAT. */
public static class FloatWriter extends ArrowFieldWriter {
- public FloatWriter(FieldVector fieldVector) {
- super(fieldVector);
+ public FloatWriter(FieldVector fieldVector, boolean isNullable) {
+ super(fieldVector, isNullable);
}
@Override
@@ -355,8 +356,8 @@ public class ArrowFieldWriters {
/** Writer for DOUBLE. */
public static class DoubleWriter extends ArrowFieldWriter {
- public DoubleWriter(FieldVector fieldVector) {
- super(fieldVector);
+ public DoubleWriter(FieldVector fieldVector, boolean isNullable) {
+ super(fieldVector, isNullable);
}
@Override
@@ -386,8 +387,8 @@ public class ArrowFieldWriters {
/** Writer for DATE. */
public static class DateWriter extends ArrowFieldWriter {
- public DateWriter(FieldVector fieldVector) {
- super(fieldVector);
+ public DateWriter(FieldVector fieldVector, boolean isNullable) {
+ super(fieldVector, isNullable);
}
@Override
@@ -417,8 +418,8 @@ public class ArrowFieldWriters {
/** Writer for TIME. */
public static class TimeWriter extends ArrowFieldWriter {
- public TimeWriter(FieldVector fieldVector) {
- super(fieldVector);
+ public TimeWriter(FieldVector fieldVector, boolean isNullable) {
+ super(fieldVector, isNullable);
}
@Override
@@ -452,8 +453,11 @@ public class ArrowFieldWriters {
@Nullable private final ZoneId castZoneId;
public TimestampWriter(
- FieldVector fieldVector, int precision, @Nullable ZoneId
castZoneId) {
- super(fieldVector);
+ FieldVector fieldVector,
+ int precision,
+ @Nullable ZoneId castZoneId,
+ boolean isNullable) {
+ super(fieldVector, isNullable);
this.precision = precision;
this.castZoneId = castZoneId;
}
@@ -494,8 +498,9 @@ public class ArrowFieldWriters {
private int offset;
- public ArrayWriter(FieldVector fieldVector, ArrowFieldWriter
elementWriter) {
- super(fieldVector);
+ public ArrayWriter(
+ FieldVector fieldVector, ArrowFieldWriter elementWriter,
boolean isNullable) {
+ super(fieldVector, isNullable);
this.elementWriter = elementWriter;
}
@@ -577,8 +582,11 @@ public class ArrowFieldWriters {
private int offset;
public MapWriter(
- FieldVector fieldVector, ArrowFieldWriter keyWriter,
ArrowFieldWriter valueWriter) {
- super(fieldVector);
+ FieldVector fieldVector,
+ ArrowFieldWriter keyWriter,
+ ArrowFieldWriter valueWriter,
+ boolean isNullable) {
+ super(fieldVector, isNullable);
this.keyWriter = keyWriter;
this.valueWriter = valueWriter;
}
@@ -735,8 +743,9 @@ public class ArrowFieldWriters {
private final ArrowFieldWriter[] fieldWriters;
- public RowWriter(FieldVector fieldVector, ArrowFieldWriter[]
fieldWriters) {
- super(fieldVector);
+ public RowWriter(
+ FieldVector fieldVector, ArrowFieldWriter[] fieldWriters,
boolean isNullable) {
+ super(fieldVector, isNullable);
this.fieldWriters = fieldWriters;
}
