Fokko commented on code in PR #3876:
URL: https://github.com/apache/parquet-java/pull/3876#discussion_r4455578798
##########
parquet-column/src/main/java/org/apache/parquet/schema/MessageType.java:
##########
@@ -34,6 +34,7 @@ public final class MessageType extends GroupType {
*/
public MessageType(String name, Type... fields) {
super(Repetition.REPEATED, name, fields);
+ TypeUtil.checkValidFiniteVectorSchema(this);
Review Comment:
I still think this isn't the right place, let's do something like:
```
diff --git
a/parquet-column/src/main/java/org/apache/parquet/schema/MessageType.java
b/parquet-column/src/main/java/org/apache/parquet/schema/MessageType.java
index 790174557..1aba0cc6a 100644
--- a/parquet-column/src/main/java/org/apache/parquet/schema/MessageType.java
+++ b/parquet-column/src/main/java/org/apache/parquet/schema/MessageType.java
@@ -34,7 +34,6 @@ public final class MessageType extends GroupType {
*/
public MessageType(String name, Type... fields) {
super(Repetition.REPEATED, name, fields);
- TypeUtil.checkValidFiniteVectorSchema(this);
}
/**
@@ -43,7 +42,6 @@ public final class MessageType extends GroupType {
*/
public MessageType(String name, List<Type> fields) {
super(Repetition.REPEATED, name, fields);
- TypeUtil.checkValidFiniteVectorSchema(this);
}
/**
diff --git
a/parquet-column/src/main/java/org/apache/parquet/schema/TypeUtil.java
b/parquet-column/src/main/java/org/apache/parquet/schema/TypeUtil.java
index c18a5cef7..b32efce39 100644
--- a/parquet-column/src/main/java/org/apache/parquet/schema/TypeUtil.java
+++ b/parquet-column/src/main/java/org/apache/parquet/schema/TypeUtil.java
@@ -18,29 +18,9 @@
*/
package org.apache.parquet.schema;
-import
org.apache.parquet.schema.LogicalTypeAnnotation.DecimalLogicalTypeAnnotation;
import
org.apache.parquet.schema.LogicalTypeAnnotation.FiniteVectorLogicalTypeAnnotation;
-import
org.apache.parquet.schema.LogicalTypeAnnotation.Float16LogicalTypeAnnotation;
-import
org.apache.parquet.schema.LogicalTypeAnnotation.IntLogicalTypeAnnotation;
-import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
-import org.apache.parquet.schema.Type.Repetition;
public class TypeUtil {
- public static boolean checkValidFiniteVectorSchema(Type type) {
- boolean found = type.getLogicalTypeAnnotation() instanceof
FiniteVectorLogicalTypeAnnotation;
- if (found) {
- if (type.isPrimitive()) {
- throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " +
type);
- }
- checkVector(type.asGroupType());
- } else if (!type.isPrimitive()) {
- for (Type child : type.asGroupType().getFields()) {
- found |= checkValidFiniteVectorSchema(child);
- }
- }
- return found;
- }
-
public static boolean hasFiniteVector(Type type) {
if (type.getLogicalTypeAnnotation() instanceof
FiniteVectorLogicalTypeAnnotation) return true;
if (!type.isPrimitive()) {
@@ -51,57 +31,6 @@ public class TypeUtil {
return false;
}
- private static void checkVector(GroupType vector) {
- if (vector.getRepetition() == Repetition.REPEATED ||
vector.getFieldCount() != 1) {
- throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " +
vector);
- }
- Type listType = vector.getType(0);
- if (listType.isPrimitive()
- || !listType.getName().equals("list")
- || listType.getRepetition() != Repetition.REPEATED
- || listType.getLogicalTypeAnnotation() != null) {
- throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " +
vector);
- }
- GroupType list = listType.asGroupType();
- if (list.getFieldCount() != 1) {
- throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " +
vector);
- }
- Type elementType = list.getType(0);
- if (!elementType.isPrimitive()
- || !elementType.getName().equals("element")
- || elementType.getRepetition() != Repetition.REQUIRED) {
- throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " +
vector);
- }
- PrimitiveType element = elementType.asPrimitiveType();
- PrimitiveTypeName physical = element.getPrimitiveTypeName();
- LogicalTypeAnnotation logical = element.getLogicalTypeAnnotation();
- boolean allowed = switch (physical) {
- case BOOLEAN, FLOAT, DOUBLE -> logical == null;
- case INT32, INT64 ->
- logical == null
- || logical instanceof IntLogicalTypeAnnotation
- || logical instanceof DecimalLogicalTypeAnnotation;
- case FIXED_LEN_BYTE_ARRAY ->
- logical instanceof Float16LogicalTypeAnnotation || logical
instanceof DecimalLogicalTypeAnnotation;
- default -> false;
- };
- if (!allowed) {
- throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " +
vector);
- }
- try {
- Types.PrimitiveBuilder<PrimitiveType> builder =
Types.primitive(physical, element.getRepetition());
- if (physical == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) {
- builder.length(element.getTypeLength());
- }
- if (logical != null) {
- builder.as(logical);
- }
- builder.named(element.getName());
- } catch (IllegalArgumentException | IllegalStateException e) {
- throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " +
vector, e);
- }
- }
-
public static FiniteVectorLogicalTypeAnnotation
getFiniteVectorType(GroupType schema, String[] path) {
if (path.length < 3) {
return null;
diff --git
a/parquet-column/src/main/java/org/apache/parquet/schema/Types.java
b/parquet-column/src/main/java/org/apache/parquet/schema/Types.java
index 2d6f0cbf8..a4b1381db 100644
--- a/parquet-column/src/main/java/org/apache/parquet/schema/Types.java
+++ b/parquet-column/src/main/java/org/apache/parquet/schema/Types.java
@@ -29,6 +29,10 @@ import java.util.Optional;
import java.util.stream.Collectors;
import org.apache.parquet.Preconditions;
import org.apache.parquet.schema.ColumnOrder.ColumnOrderName;
+import
org.apache.parquet.schema.LogicalTypeAnnotation.DecimalLogicalTypeAnnotation;
+import
org.apache.parquet.schema.LogicalTypeAnnotation.FiniteVectorLogicalTypeAnnotation;
+import
org.apache.parquet.schema.LogicalTypeAnnotation.Float16LogicalTypeAnnotation;
+import
org.apache.parquet.schema.LogicalTypeAnnotation.IntLogicalTypeAnnotation;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
import org.apache.parquet.schema.Type.ID;
import org.slf4j.Logger;
@@ -846,10 +850,50 @@ public class Types {
@Override
protected GroupType build(String name) {
- if (newLogicalTypeSet) {
- return new GroupType(repetition, name, logicalTypeAnnotation,
fields, id);
- } else {
- return new GroupType(repetition, name, getOriginalType(), fields,
id);
+ GroupType group = newLogicalTypeSet
+ ? new GroupType(repetition, name, logicalTypeAnnotation, fields,
id)
+ : new GroupType(repetition, name, getOriginalType(), fields, id);
+ if (logicalTypeAnnotation instanceof
FiniteVectorLogicalTypeAnnotation) {
+ checkFiniteVector(group);
+ }
+ return group;
+ }
+
+ /**
+ * FINITE_VECTOR must annotate the canonical 3-level LIST structure
with a
+ * required numeric or boolean element.
+ */
+ private static void checkFiniteVector(GroupType vector) {
+ if (vector.isRepetition(Type.Repetition.REPEATED) ||
vector.getFieldCount() != 1) {
+ throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " +
vector);
+ }
+ Type list = vector.getType(0);
+ if (list.isPrimitive()
+ || !list.getName().equals("list")
+ || !list.isRepetition(Type.Repetition.REPEATED)
+ || list.getLogicalTypeAnnotation() != null
+ || list.asGroupType().getFieldCount() != 1) {
+ throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " +
vector);
+ }
+ Type element = list.asGroupType().getType(0);
+ if (!element.isPrimitive()
+ || !element.getName().equals("element")
+ || !element.isRepetition(Type.Repetition.REQUIRED)) {
+ throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " +
vector);
+ }
+ LogicalTypeAnnotation logical = element.getLogicalTypeAnnotation();
+ boolean allowed = switch
(element.asPrimitiveType().getPrimitiveTypeName()) {
+ case BOOLEAN, FLOAT, DOUBLE -> logical == null;
+ case INT32, INT64 ->
+ logical == null
+ || logical instanceof IntLogicalTypeAnnotation
+ || logical instanceof DecimalLogicalTypeAnnotation;
+ case FIXED_LEN_BYTE_ARRAY ->
+ logical instanceof Float16LogicalTypeAnnotation || logical
instanceof DecimalLogicalTypeAnnotation;
+ default -> false;
+ };
+ if (!allowed) {
+ throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " +
vector);
}
}
diff --git
a/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuildersWithLogicalTypes.java
b/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuildersWithLogicalTypes.java
index 61507cc2c..a6ebf5c9a 100644
---
a/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuildersWithLogicalTypes.java
+++
b/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuildersWithLogicalTypes.java
@@ -45,7 +45,6 @@ import static
org.apache.parquet.schema.Type.Repetition.REQUIRED;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import java.util.Collections;
import java.util.Optional;
import java.util.stream.Stream;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
@@ -144,33 +143,48 @@ public class TestTypeBuildersWithLogicalTypes {
vector(Types.required(FIXED_LEN_BYTE_ARRAY).length(2).named("element"))),
Arguments.of(vector(Types.required(INT96).named("element"))),
Arguments.of(vector(Types.required(BINARY).as(stringType()).named("element"))),
-
Arguments.of(vector(Types.required(INT32).as(dateType()).named("element"))),
- Arguments.of(
-
vector(Types.required(INT32).named("element").withLogicalTypeAnnotation(intType(64,
true)))));
+
Arguments.of(vector(Types.required(INT32).as(dateType()).named("element"))));
}
@ParameterizedTest
@MethodSource("invalidVectors")
public void testInvalidVectorSchema(GroupType vector) {
- assertThatThrownBy(() -> new MessageType("m", vector))
+ assertThatThrownBy(() -> Types.buildGroup(vector.getRepetition())
+ .as(vector.getLogicalTypeAnnotation())
+ .addFields(vector.getFields().toArray(new Type[0]))
+ .named(vector.getName()))
.isInstanceOf(InvalidSchemaException.class)
.hasMessage("Invalid FINITE_VECTOR schema: " + vector);
}
private static Stream<Executable> vectorConstructionBoundaries() {
- Type element = Types.optional(FLOAT).named("element");
- GroupType invalid = vector(element);
- GroupType bad = new GroupType(
- REQUIRED, "bad", finiteVectorType(5), new
GroupType(Repetition.REPEATED, "list", element));
+ GroupType list =
+ new GroupType(Repetition.REPEATED, "list",
Types.optional(FLOAT).named("element"));
return Stream.of(
- () -> new MessageType("m", Collections.singletonList(invalid)),
- () -> Types.buildMessage().addField(invalid).named("m"),
+ () -> Types.requiredGroup()
+ .as(finiteVectorType(5))
+ .addField(list)
+ .named("v"),
+ () -> Types.buildMessage()
+ .requiredGroup()
+ .as(finiteVectorType(5))
+ .repeatedGroup()
+ .optional(FLOAT)
+ .named("element")
+ .named("list")
+ .named("v")
+ .named("m"),
+ () -> Types.buildMessage()
+ .requiredGroup()
+ .addField(vector(Types.required(FLOAT).named("element")))
+ .requiredGroup()
+ .as(finiteVectorType(5))
+ .addField(list)
+ .named("bad")
+ .named("parent")
+ .named("m"),
() -> MessageTypeParser.parseMessageType("message m { required
group v (FINITE_VECTOR(5)) { "
- + "repeated group list { optional float element; } } }"),
- () -> new MessageType(
- "m",
- new GroupType(
- REQUIRED, "parent",
vector(Types.required(FLOAT).named("element")), bad)));
+ + "repeated group list { optional float element; } } }"));
}
@ParameterizedTest
diff --git
a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
index 61cd86707..4e1aa1749 100644
---
a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
+++
b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
@@ -2036,8 +2036,8 @@ public class ParquetMetadataConverter {
builder.id(root.field_id);
}
Iterator<ColumnOrder> columnOrderIterator = columnOrders == null ? null
: columnOrders.iterator();
- buildChildren(builder, iterator, root.getNum_children(),
columnOrderIterator);
try {
+ buildChildren(builder, iterator, root.getNum_children(),
columnOrderIterator);
return builder.named(root.name);
} catch (InvalidSchemaException e) {
throw new ParquetDecodingException("Invalid FINITE_VECTOR schema in
file metadata", e);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]