stevenzwu commented on code in PR #9606:
URL: https://github.com/apache/iceberg/pull/9606#discussion_r1509462559
##########
flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkSchemaUtil.java:
##########
@@ -64,26 +68,75 @@ public static Schema convert(TableSchema schema) {
RowType root = (RowType) schemaType;
Type converted = root.accept(new FlinkTypeToType(root));
- Schema iSchema = new Schema(converted.asStructType().fields());
- return freshIdentifierFieldIds(iSchema, schema);
+ Schema icebergSchema = new Schema(converted.asStructType().fields());
+ return freshIdentifierFieldIds(icebergSchema, schema);
}
- private static Schema freshIdentifierFieldIds(Schema iSchema, TableSchema
schema) {
+ /** Convert the flink table schema to apache iceberg schema with column
comment. */
+ public static Schema convert(ResolvedSchema flinkSchema) {
+ List<Column> tableColumns = flinkSchema.getColumns();
+ // copy from org.apache.flink.table.api.Schema#toRowDataType
+ DataTypes.Field[] fields =
+ tableColumns.stream()
+ .map(
+ column -> {
+ if (column.getComment().isPresent()) {
+ return DataTypes.FIELD(
+ column.getName(), column.getDataType(),
column.getComment().get());
+ } else {
+ return DataTypes.FIELD(column.getName(),
column.getDataType());
+ }
+ })
+ .toArray(DataTypes.Field[]::new);
+
+ LogicalType schemaType = DataTypes.ROW(fields).notNull().getLogicalType();
+ Preconditions.checkArgument(
+ schemaType instanceof RowType, "Schema logical type should be
RowType.");
+
+ RowType root = (RowType) schemaType;
+ Type converted = root.accept(new FlinkTypeToType(root));
+ Schema icebergSchema = new Schema(converted.asStructType().fields());
+ return freshIdentifierFieldIds(icebergSchema, flinkSchema);
+ }
+
+ /** @deprecated Use {@link #freshIdentifierFieldIds(Schema, ResolvedSchema)}
instead. */
+ @Deprecated
+ private static Schema freshIdentifierFieldIds(Schema icebergSchema,
TableSchema flinkSchema) {
Review Comment:
the diff looks a little odd to me :) the old function looks as new code,
while the function appears to be new code. just cosmetic thing though
##########
flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkSchemaUtil.java:
##########
@@ -64,26 +68,75 @@ public static Schema convert(TableSchema schema) {
RowType root = (RowType) schemaType;
Type converted = root.accept(new FlinkTypeToType(root));
- Schema iSchema = new Schema(converted.asStructType().fields());
- return freshIdentifierFieldIds(iSchema, schema);
+ Schema icebergSchema = new Schema(converted.asStructType().fields());
+ return freshIdentifierFieldIds(icebergSchema, schema);
}
- private static Schema freshIdentifierFieldIds(Schema iSchema, TableSchema
schema) {
+ /** Convert the flink table schema to apache iceberg schema with column
comment. */
+ public static Schema convert(ResolvedSchema flinkSchema) {
+ List<Column> tableColumns = flinkSchema.getColumns();
+ // copy from org.apache.flink.table.api.Schema#toRowDataType
+ DataTypes.Field[] fields =
+ tableColumns.stream()
+ .map(
+ column -> {
+ if (column.getComment().isPresent()) {
+ return DataTypes.FIELD(
+ column.getName(), column.getDataType(),
column.getComment().get());
+ } else {
+ return DataTypes.FIELD(column.getName(),
column.getDataType());
+ }
+ })
+ .toArray(DataTypes.Field[]::new);
+
+ LogicalType schemaType = DataTypes.ROW(fields).notNull().getLogicalType();
+ Preconditions.checkArgument(
+ schemaType instanceof RowType, "Schema logical type should be
RowType.");
+
+ RowType root = (RowType) schemaType;
+ Type converted = root.accept(new FlinkTypeToType(root));
+ Schema icebergSchema = new Schema(converted.asStructType().fields());
+ return freshIdentifierFieldIds(icebergSchema, flinkSchema);
+ }
+
+ /** @deprecated Use {@link #freshIdentifierFieldIds(Schema, ResolvedSchema)}
instead. */
+ @Deprecated
+ private static Schema freshIdentifierFieldIds(Schema icebergSchema,
TableSchema flinkSchema) {
+ // Locate the identifier field id list.
+ Set<Integer> identifierFieldIds = Sets.newHashSet();
+ if (flinkSchema.getPrimaryKey().isPresent()) {
+ for (String column : flinkSchema.getPrimaryKey().get().getColumns()) {
+ Types.NestedField field = icebergSchema.findField(column);
+ Preconditions.checkNotNull(
+ field,
+ "Cannot find field ID for the primary key column %s in flinkSchema
%s",
Review Comment:
same as the comment below for `schema` in error msg.
##########
flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkSchemaUtil.java:
##########
@@ -64,26 +68,75 @@ public static Schema convert(TableSchema schema) {
RowType root = (RowType) schemaType;
Type converted = root.accept(new FlinkTypeToType(root));
- Schema iSchema = new Schema(converted.asStructType().fields());
- return freshIdentifierFieldIds(iSchema, schema);
+ Schema icebergSchema = new Schema(converted.asStructType().fields());
+ return freshIdentifierFieldIds(icebergSchema, schema);
}
- private static Schema freshIdentifierFieldIds(Schema iSchema, TableSchema
schema) {
+ /** Convert the flink table schema to apache iceberg schema with column
comment. */
+ public static Schema convert(ResolvedSchema flinkSchema) {
+ List<Column> tableColumns = flinkSchema.getColumns();
+ // copy from org.apache.flink.table.api.Schema#toRowDataType
+ DataTypes.Field[] fields =
+ tableColumns.stream()
+ .map(
+ column -> {
+ if (column.getComment().isPresent()) {
+ return DataTypes.FIELD(
+ column.getName(), column.getDataType(),
column.getComment().get());
+ } else {
+ return DataTypes.FIELD(column.getName(),
column.getDataType());
+ }
+ })
+ .toArray(DataTypes.Field[]::new);
+
+ LogicalType schemaType = DataTypes.ROW(fields).notNull().getLogicalType();
+ Preconditions.checkArgument(
+ schemaType instanceof RowType, "Schema logical type should be
RowType.");
+
+ RowType root = (RowType) schemaType;
+ Type converted = root.accept(new FlinkTypeToType(root));
+ Schema icebergSchema = new Schema(converted.asStructType().fields());
+ return freshIdentifierFieldIds(icebergSchema, flinkSchema);
+ }
+
+ /** @deprecated Use {@link #freshIdentifierFieldIds(Schema, ResolvedSchema)}
instead. */
+ @Deprecated
Review Comment:
nit: this is a private method. not sure if @Deprecated is needed here
##########
flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkSchemaUtil.java:
##########
@@ -64,26 +68,75 @@ public static Schema convert(TableSchema schema) {
RowType root = (RowType) schemaType;
Type converted = root.accept(new FlinkTypeToType(root));
- Schema iSchema = new Schema(converted.asStructType().fields());
- return freshIdentifierFieldIds(iSchema, schema);
+ Schema icebergSchema = new Schema(converted.asStructType().fields());
+ return freshIdentifierFieldIds(icebergSchema, schema);
}
- private static Schema freshIdentifierFieldIds(Schema iSchema, TableSchema
schema) {
+ /** Convert the flink table schema to apache iceberg schema with column
comment. */
+ public static Schema convert(ResolvedSchema flinkSchema) {
+ List<Column> tableColumns = flinkSchema.getColumns();
+ // copy from org.apache.flink.table.api.Schema#toRowDataType
+ DataTypes.Field[] fields =
+ tableColumns.stream()
+ .map(
+ column -> {
+ if (column.getComment().isPresent()) {
+ return DataTypes.FIELD(
+ column.getName(), column.getDataType(),
column.getComment().get());
+ } else {
+ return DataTypes.FIELD(column.getName(),
column.getDataType());
+ }
+ })
+ .toArray(DataTypes.Field[]::new);
+
+ LogicalType schemaType = DataTypes.ROW(fields).notNull().getLogicalType();
+ Preconditions.checkArgument(
+ schemaType instanceof RowType, "Schema logical type should be
RowType.");
+
+ RowType root = (RowType) schemaType;
+ Type converted = root.accept(new FlinkTypeToType(root));
+ Schema icebergSchema = new Schema(converted.asStructType().fields());
+ return freshIdentifierFieldIds(icebergSchema, flinkSchema);
+ }
+
+ /** @deprecated Use {@link #freshIdentifierFieldIds(Schema, ResolvedSchema)}
instead. */
+ @Deprecated
+ private static Schema freshIdentifierFieldIds(Schema icebergSchema,
TableSchema flinkSchema) {
+ // Locate the identifier field id list.
+ Set<Integer> identifierFieldIds = Sets.newHashSet();
+ if (flinkSchema.getPrimaryKey().isPresent()) {
+ for (String column : flinkSchema.getPrimaryKey().get().getColumns()) {
+ Types.NestedField field = icebergSchema.findField(column);
+ Preconditions.checkNotNull(
+ field,
+ "Cannot find field ID for the primary key column %s in flinkSchema
%s",
+ column,
+ icebergSchema);
+ identifierFieldIds.add(field.fieldId());
+ }
+ }
+
+ return new Schema(
+ icebergSchema.schemaId(), icebergSchema.asStruct().fields(),
identifierFieldIds);
+ }
+
+ private static Schema freshIdentifierFieldIds(Schema icebergSchema,
ResolvedSchema flinkSchema) {
// Locate the identifier field id list.
Set<Integer> identifierFieldIds = Sets.newHashSet();
- if (schema.getPrimaryKey().isPresent()) {
- for (String column : schema.getPrimaryKey().get().getColumns()) {
- Types.NestedField field = iSchema.findField(column);
+ if (flinkSchema.getPrimaryKey().isPresent()) {
+ for (String column : flinkSchema.getPrimaryKey().get().getColumns()) {
+ Types.NestedField field = icebergSchema.findField(column);
Preconditions.checkNotNull(
field,
- "Cannot find field ID for the primary key column %s in schema %s",
+ "Cannot find field ID for the primary key column %s in flinkSchema
%s",
Review Comment:
this seems incorrect. I think the old comment is correct, `schema` implied
`Iceberg table schema`.
also style is to avoid variable name in error message, which is not user
friendly. hence it would have been `Flink schema`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
