This is an automated email from the ASF dual-hosted git repository.
jerryshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 5225253e7d [#13273] fix(core): batch table column metadata inserts
(#13277)
5225253e7d is described below
commit 5225253e7d6f738fccde9a76072ddb151f9c53e3
Author: roryqi <[email protected]>
AuthorDate: Thu Sep 17 20:29:30 2026 +0800
[#13273] fix(core): batch table column metadata inserts (#13277)
### What changes were proposed in this pull request?
- Split table column metadata inserts into batches of 1,000 columns.
- Use the batched insertion path for both table creation and table
updates.
- Add a regression test that stores and reads back a table with 5,000
columns.
### Why are the changes needed?
The current implementation inserts all column metadata using one
multi-value SQL statement. Each column uses 16 bound parameters, so
tables with 4,096 or more columns exceed PostgreSQL's 65,535-parameter
limit.
Executing bounded batches inside the existing table transaction avoids
the parameter limit while preserving atomicity.
Fix: #13273
### Does this PR introduce _any_ user-facing change?
Yes. Tables with more than 4,095 columns can now be created when
PostgreSQL is used as the relational metadata store.
There are no API or configuration changes.
### How was this patch tested?
Ran the table column metadata test suite against H2, MySQL, and
PostgreSQL:
./gradlew :core:test --tests
org.apache.gravitino.storage.relational.service.TestTableColumnMetaService
-PskipDockerTests=false
All 24 tests passed, including the new 5,000-column regression test.
---
.../relational/service/TableColumnMetaService.java | 18 +++++++----
.../service/TestTableColumnMetaService.java | 37 ++++++++++++++++++++++
2 files changed, 49 insertions(+), 6 deletions(-)
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableColumnMetaService.java
b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableColumnMetaService.java
index 1433f7ac62..925c12e87d 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableColumnMetaService.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableColumnMetaService.java
@@ -40,6 +40,7 @@ import
org.apache.gravitino.storage.relational.utils.SessionUtils;
public class TableColumnMetaService {
+ private static final int COLUMN_INSERT_BATCH_SIZE = 1000;
private static final TableColumnMetaService INSTANCE = new
TableColumnMetaService();
private TableColumnMetaService() {}
@@ -107,9 +108,7 @@ public class TableColumnMetaService {
List<ColumnPO> columnPOs =
POConverters.initializeColumnPOs(tablePO, columnEntities,
ColumnPO.ColumnOpType.CREATE);
- // insertColumnPOs will be done in insertTable transaction, so we don't do
commit here.
- SessionUtils.doWithoutCommit(
- TableColumnMapper.class, mapper -> mapper.insertColumnPOs(columnPOs));
+ insertColumnPOsInBatches(columnPOs);
}
@Monitored(
@@ -194,8 +193,15 @@ public class TableColumnMetaService {
return;
}
- // updateColumns will be done in updateTable transaction, so we don't do
commit here.
- SessionUtils.doWithoutCommit(
- TableColumnMapper.class, mapper ->
mapper.insertColumnPOs(columnPOsToInsert));
+ insertColumnPOsInBatches(columnPOsToInsert);
+ }
+
+ private void insertColumnPOsInBatches(List<ColumnPO> columnPOs) {
+ // Column inserts run inside the table transaction, so no batch commits
independently.
+ Lists.partition(columnPOs, COLUMN_INSERT_BATCH_SIZE)
+ .forEach(
+ batch ->
+ SessionUtils.doWithoutCommit(
+ TableColumnMapper.class, mapper ->
mapper.insertColumnPOs(batch)));
}
}
diff --git
a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestTableColumnMetaService.java
b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestTableColumnMetaService.java
index 057a12ac16..bf406b5a3a 100644
---
a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestTableColumnMetaService.java
+++
b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestTableColumnMetaService.java
@@ -46,6 +46,7 @@ import org.junit.jupiter.api.TestTemplate;
public class TestTableColumnMetaService extends TestJDBCBackend {
+ private static final int WIDE_TABLE_COLUMN_COUNT = 5000;
private static final String METALAKE_NAME = "metalake_for_table_column_test";
@TestTemplate
@@ -149,6 +150,42 @@ public class TestTableColumnMetaService extends
TestJDBCBackend {
compareTwoColumns(createdTable3.columns(), retrievedTable3.columns());
}
+ @TestTemplate
+ public void testInsertWideTableColumnsInBatches() throws IOException {
+ String catalogName = "catalog1";
+ String schemaName = "schema1";
+ createParentEntities(METALAKE_NAME, catalogName, schemaName, AUDIT_INFO);
+
+ List<ColumnEntity> columns = new ArrayList<>(WIDE_TABLE_COLUMN_COUNT);
+ for (int i = 0; i < WIDE_TABLE_COLUMN_COUNT; i++) {
+ columns.add(
+ ColumnEntity.builder()
+ .withId(RandomIdGenerator.INSTANCE.nextId())
+ .withName("column_" + i)
+ .withPosition(i)
+ .withDataType(Types.IntegerType.get())
+ .withNullable(true)
+ .withAutoIncrement(false)
+ .withAuditInfo(AUDIT_INFO)
+ .build());
+ }
+
+ TableEntity createdTable =
+ TableEntity.builder()
+ .withId(RandomIdGenerator.INSTANCE.nextId())
+ .withName("wide_table")
+ .withNamespace(Namespace.of(METALAKE_NAME, catalogName,
schemaName))
+ .withColumns(columns)
+ .withAuditInfo(AUDIT_INFO)
+ .build();
+
+ TableMetaService.getInstance().insertTable(createdTable, false);
+
+ TableEntity retrievedTable =
+
TableMetaService.getInstance().getTableByIdentifier(createdTable.nameIdentifier());
+ compareTwoColumns(createdTable.columns(), retrievedTable.columns());
+ }
+
@TestTemplate
public void testUpdateTable() throws IOException {
String catalogName = "catalog1";
