hailin0 commented on code in PR #2747:
URL:
https://github.com/apache/incubator-seatunnel/pull/2747#discussion_r973999253
##########
seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/OrcReadStrategy.java:
##########
@@ -163,6 +167,73 @@ boolean checkFileType(String path) {
}
}
+ private SeaTunnelDataType<?> orcDataType2SeaTunnelDataType(TypeDescription
typeDescription) {
+ switch (typeDescription.getCategory()) {
+ case BOOLEAN:
+ return BasicType.BOOLEAN_TYPE;
+ case INT:
+ return BasicType.INT_TYPE;
+ case BYTE:
+ case SHORT:
Review Comment:
Why not use `TINYINT`、`SMALLINT`?
##########
seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/OrcReadStrategy.java:
##########
@@ -163,6 +167,73 @@ boolean checkFileType(String path) {
}
}
+ private SeaTunnelDataType<?> orcDataType2SeaTunnelDataType(TypeDescription
typeDescription) {
+ switch (typeDescription.getCategory()) {
+ case BOOLEAN:
+ return BasicType.BOOLEAN_TYPE;
+ case INT:
+ return BasicType.INT_TYPE;
+ case BYTE:
+ case SHORT:
+ case LONG:
+ return BasicType.LONG_TYPE;
+ case FLOAT:
Review Comment:
Why not use `FLOAT`?
##########
seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/OrcReadStrategy.java:
##########
@@ -163,6 +167,73 @@ boolean checkFileType(String path) {
}
}
+ private SeaTunnelDataType<?> orcDataType2SeaTunnelDataType(TypeDescription
typeDescription) {
+ switch (typeDescription.getCategory()) {
+ case BOOLEAN:
+ return BasicType.BOOLEAN_TYPE;
+ case INT:
+ return BasicType.INT_TYPE;
+ case BYTE:
+ case SHORT:
+ case LONG:
+ return BasicType.LONG_TYPE;
+ case FLOAT:
+ case DOUBLE:
+ return BasicType.DOUBLE_TYPE;
+ case STRING:
+ case BINARY:
+ case VARCHAR:
+ case CHAR:
+ return BasicType.STRING_TYPE;
+ case DATE:
+ return LocalTimeType.LOCAL_DATE_TYPE;
+ case TIMESTAMP:
+ return LocalTimeType.LOCAL_DATE_TIME_TYPE;
+ case DECIMAL:
+ int precision = typeDescription.getPrecision();
+ int scale = typeDescription.getScale();
+ return new DecimalType(precision, scale);
+ case LIST:
+ TypeDescription listType =
typeDescription.getChildren().get(0);
+ SeaTunnelDataType<?> seaTunnelDataType =
orcDataType2SeaTunnelDataType(listType);
+ if (BasicType.STRING_TYPE.equals(seaTunnelDataType)) {
+ return ArrayType.STRING_ARRAY_TYPE;
+ } else if (BasicType.BOOLEAN_TYPE.equals(seaTunnelDataType)) {
+ return ArrayType.BOOLEAN_ARRAY_TYPE;
+ } else if (BasicType.BYTE_TYPE.equals(seaTunnelDataType)) {
+ return ArrayType.BYTE_ARRAY_TYPE;
+ } else if (BasicType.SHORT_TYPE.equals(seaTunnelDataType)) {
+ return ArrayType.SHORT_ARRAY_TYPE;
+ } else if (BasicType.INT_TYPE.equals(seaTunnelDataType)) {
+ return ArrayType.INT_ARRAY_TYPE;
+ } else if (BasicType.LONG_TYPE.equals(seaTunnelDataType)) {
+ return ArrayType.LONG_ARRAY_TYPE;
+ } else if (BasicType.FLOAT_TYPE.equals(seaTunnelDataType)) {
+ return ArrayType.FLOAT_ARRAY_TYPE;
+ } else if (BasicType.DOUBLE_TYPE.equals(seaTunnelDataType)) {
+ return ArrayType.DOUBLE_ARRAY_TYPE;
+ }
+ String errorMsg = String.format("SeaTunnel array type not
supported this genericType [%s] yet", seaTunnelDataType);
+ throw new RuntimeException(errorMsg);
Review Comment:
Use `switch (seaTunnelDataType.getSqlType())`?
##########
seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/OrcReadStrategy.java:
##########
@@ -163,6 +167,73 @@ boolean checkFileType(String path) {
}
}
+ private SeaTunnelDataType<?> orcDataType2SeaTunnelDataType(TypeDescription
typeDescription) {
+ switch (typeDescription.getCategory()) {
+ case BOOLEAN:
+ return BasicType.BOOLEAN_TYPE;
+ case INT:
+ return BasicType.INT_TYPE;
+ case BYTE:
+ case SHORT:
+ case LONG:
+ return BasicType.LONG_TYPE;
+ case FLOAT:
+ case DOUBLE:
+ return BasicType.DOUBLE_TYPE;
+ case STRING:
+ case BINARY:
Review Comment:
why not use seatunnel `BYTES`?
##########
seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/OrcReadStrategy.java:
##########
@@ -163,6 +167,73 @@ boolean checkFileType(String path) {
}
}
+ private SeaTunnelDataType<?> orcDataType2SeaTunnelDataType(TypeDescription
typeDescription) {
+ switch (typeDescription.getCategory()) {
+ case BOOLEAN:
+ return BasicType.BOOLEAN_TYPE;
+ case INT:
+ return BasicType.INT_TYPE;
+ case BYTE:
+ case SHORT:
+ case LONG:
+ return BasicType.LONG_TYPE;
+ case FLOAT:
+ case DOUBLE:
+ return BasicType.DOUBLE_TYPE;
+ case STRING:
+ case BINARY:
+ case VARCHAR:
+ case CHAR:
+ return BasicType.STRING_TYPE;
+ case DATE:
+ return LocalTimeType.LOCAL_DATE_TYPE;
+ case TIMESTAMP:
+ return LocalTimeType.LOCAL_DATE_TIME_TYPE;
+ case DECIMAL:
+ int precision = typeDescription.getPrecision();
+ int scale = typeDescription.getScale();
+ return new DecimalType(precision, scale);
+ case LIST:
+ TypeDescription listType =
typeDescription.getChildren().get(0);
+ SeaTunnelDataType<?> seaTunnelDataType =
orcDataType2SeaTunnelDataType(listType);
+ if (BasicType.STRING_TYPE.equals(seaTunnelDataType)) {
+ return ArrayType.STRING_ARRAY_TYPE;
+ } else if (BasicType.BOOLEAN_TYPE.equals(seaTunnelDataType)) {
+ return ArrayType.BOOLEAN_ARRAY_TYPE;
+ } else if (BasicType.BYTE_TYPE.equals(seaTunnelDataType)) {
+ return ArrayType.BYTE_ARRAY_TYPE;
+ } else if (BasicType.SHORT_TYPE.equals(seaTunnelDataType)) {
+ return ArrayType.SHORT_ARRAY_TYPE;
+ } else if (BasicType.INT_TYPE.equals(seaTunnelDataType)) {
+ return ArrayType.INT_ARRAY_TYPE;
+ } else if (BasicType.LONG_TYPE.equals(seaTunnelDataType)) {
+ return ArrayType.LONG_ARRAY_TYPE;
+ } else if (BasicType.FLOAT_TYPE.equals(seaTunnelDataType)) {
+ return ArrayType.FLOAT_ARRAY_TYPE;
+ } else if (BasicType.DOUBLE_TYPE.equals(seaTunnelDataType)) {
+ return ArrayType.DOUBLE_ARRAY_TYPE;
+ }
+ String errorMsg = String.format("SeaTunnel array type not
supported this genericType [%s] yet", seaTunnelDataType);
+ throw new RuntimeException(errorMsg);
+ case MAP:
+ TypeDescription keyType = typeDescription.getChildren().get(0);
+ TypeDescription valueType =
typeDescription.getChildren().get(1);
+ return new MapType<>(orcDataType2SeaTunnelDataType(keyType),
orcDataType2SeaTunnelDataType(valueType));
+ case STRUCT:
+ List<TypeDescription> children = typeDescription.getChildren();
+ String[] fieldNames =
typeDescription.getFieldNames().toArray(new String[0]);
+ SeaTunnelDataType<?>[] fieldTypes =
children.stream().map(this::orcDataType2SeaTunnelDataType).toArray(SeaTunnelDataType<?>[]::new);
+ return new SeaTunnelRowType(fieldNames, fieldTypes);
+ case UNION:
+ // TODO: How to explain this type using SeaTunnel data type
+ throw new RuntimeException("SeaTunnel not supported orc
[union] type yet");
+ default:
+ // do nothing
+ // never get in there
+ return null;
Review Comment:
You will get unexpected results if the orc is upgraded, so log error message
and throw exception
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
