This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 1a9fbc9be6 [INLONG-8173][Sort] Fix the NPE problem when adding new
columns (#8174)
1a9fbc9be6 is described below
commit 1a9fbc9be6e5e193d21ddf2f842cd47c8da53001
Author: LinChen <[email protected]>
AuthorDate: Wed Jun 7 19:47:24 2023 +0800
[INLONG-8173][Sort] Fix the NPE problem when adding new columns (#8174)
---
.../sort/iceberg/schema/IcebergSchemaChangeHelper.java | 17 +++++++++++++----
.../sink/multiple/DynamicSchemaHandleOperator.java | 9 +++++----
2 files changed, 18 insertions(+), 8 deletions(-)
diff --git
a/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/iceberg/src/main/java/org/apache/inlong/sort/iceberg/schema/IcebergSchemaChangeHelper.java
b/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/iceberg/src/main/java/org/apache/inlong/sort/iceberg/schema/IcebergSchemaChangeHelper.java
index b3abc456c3..a44aba50ad 100644
---
a/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/iceberg/src/main/java/org/apache/inlong/sort/iceberg/schema/IcebergSchemaChangeHelper.java
+++
b/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/iceberg/src/main/java/org/apache/inlong/sort/iceberg/schema/IcebergSchemaChangeHelper.java
@@ -43,6 +43,7 @@ import org.apache.iceberg.catalog.Catalog;
import org.apache.iceberg.catalog.SupportsNamespaces;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.flink.FlinkSchemaUtil;
+import org.apache.iceberg.types.Types;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -122,7 +123,7 @@ public class IcebergSchemaChangeHelper extends
SchemaChangeHelper {
}
}
- public void doAddColumn(List<AlterColumn> alterColumns, TableIdentifier
tableId) {
+ private void doAddColumn(List<AlterColumn> alterColumns, TableIdentifier
tableId) {
List<TableChange> tableChanges = new ArrayList<>();
Table table = catalog.loadTable(tableId);
Transaction transaction = table.newTransaction();
@@ -130,9 +131,17 @@ public class IcebergSchemaChangeHelper extends
SchemaChangeHelper {
alterColumns.forEach(alterColumn -> {
Column column = alterColumn.getNewColumn();
LogicalType dataType =
dynamicSchemaFormat.sqlType2FlinkType(column.getJdbcType());
- TableChange.ColumnPosition position =
- column.getPosition().getPositionType() ==
PositionType.FIRST ? TableChange.ColumnPosition.first()
- :
TableChange.ColumnPosition.after(column.getName());
+ TableChange.ColumnPosition position;
+ if (column.getPosition() != null &&
column.getPosition().getColumnName() != null) {
+ position =
+ column.getPosition().getPositionType() ==
PositionType.FIRST
+ ? TableChange.ColumnPosition.first()
+ :
TableChange.ColumnPosition.after(column.getPosition().getColumnName());
+ } else {
+ List<Types.NestedField> columns = table.schema().columns();
+ Types.NestedField lastField = columns.get(columns.size() - 1);
+ position = TableChange.ColumnPosition.after(lastField.name());
+ }
TableChange.AddColumn addColumn = new TableChange.AddColumn(new
String[]{column.getName()},
dataType, column.isNullable(), column.getComment(),
position);
tableChanges.add(addColumn);
diff --git
a/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/iceberg/src/main/java/org/apache/inlong/sort/iceberg/sink/multiple/DynamicSchemaHandleOperator.java
b/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/iceberg/src/main/java/org/apache/inlong/sort/iceberg/sink/multiple/DynamicSchemaHandleOperator.java
index 44541d132f..1b12639d57 100644
---
a/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/iceberg/src/main/java/org/apache/inlong/sort/iceberg/sink/multiple/DynamicSchemaHandleOperator.java
+++
b/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/iceberg/src/main/java/org/apache/inlong/sort/iceberg/sink/multiple/DynamicSchemaHandleOperator.java
@@ -177,10 +177,11 @@ public class DynamicSchemaHandleOperator extends
AbstractStreamOperator<RecordWi
}
this.dirtySinkHelper.open(new Configuration());
this.schemaChangeHelper = new
IcebergSchemaChangeHelper(dynamicSchemaFormat,
- enableSchemaChange, enableSchemaChange ?
SchemaChangeUtils.deserialize(schemaChangePolicies) : null,
- multipleSinkOption.getDatabasePattern(),
multipleSinkOption.getTablePattern(),
- multipleSinkOption.getSchemaUpdatePolicy(), metricData,
dirtySinkHelper,
- catalog, asNamespaceCatalog);
+ enableSchemaChange, enableSchemaChange ?
SchemaChangeUtils.deserialize(schemaChangePolicies)
+ : Collections.emptyMap(),
+ multipleSinkOption.getDatabasePattern(),
+ multipleSinkOption.getTablePattern(),
multipleSinkOption.getSchemaUpdatePolicy(), metricData,
+ dirtySinkHelper, catalog, asNamespaceCatalog);
}
@Override