This is an automated email from the ASF dual-hosted git repository. yuqi1129 pushed a commit to branch yuqi/clickhouse-partition-transforms in repository https://gitbox.apache.org/repos/asf/gravitino.git
commit 7631e459d8fa20f1d139d249d915cba3a6cdc9bd Author: yuqi <[email protected]> AuthorDate: Sat May 9 19:05:55 2026 +0800 [#11017] feat(clickhouse): Support date partition transforms --- .../operations/ClickHouseTableOperations.java | 16 ---------- .../operations/ClickHouseTableSqlUtils.java | 33 +++++++++++++------- .../integration/test/CatalogClickHouseIT.java | 3 +- .../operations/TestClickHouseTableOperations.java | 36 ++++++++++++++++++++++ 4 files changed, 60 insertions(+), 28 deletions(-) diff --git a/catalogs-contrib/catalog-jdbc-clickhouse/src/main/java/org/apache/gravitino/catalog/clickhouse/operations/ClickHouseTableOperations.java b/catalogs-contrib/catalog-jdbc-clickhouse/src/main/java/org/apache/gravitino/catalog/clickhouse/operations/ClickHouseTableOperations.java index 4c453b7e7a..fc823ef78c 100644 --- a/catalogs-contrib/catalog-jdbc-clickhouse/src/main/java/org/apache/gravitino/catalog/clickhouse/operations/ClickHouseTableOperations.java +++ b/catalogs-contrib/catalog-jdbc-clickhouse/src/main/java/org/apache/gravitino/catalog/clickhouse/operations/ClickHouseTableOperations.java @@ -419,22 +419,6 @@ public class ClickHouseTableOperations extends JdbcTableOperations { .orElse(null); } - private String toPartitionExpression(Transform transform) { - Preconditions.checkArgument(transform != null, "Partition transform cannot be null"); - Preconditions.checkArgument( - StringUtils.equalsIgnoreCase(transform.name(), Transforms.NAME_OF_IDENTITY), - "Unsupported partition transform: " + transform.name()); - Preconditions.checkArgument( - transform.arguments().length == 1 - && transform.arguments()[0] instanceof NamedReference - && ((NamedReference) transform.arguments()[0]).fieldName().length == 1, - "ClickHouse only supports single column identity partitioning"); - - String fieldName = - ((NamedReference) transform.arguments()[0]).fieldName()[0]; // already validated - return quoteIdentifier(fieldName); - } - private void validateNoAutoIncrementColumns(JdbcColumn[] columns) { if (ArrayUtils.isEmpty(columns)) { return; diff --git a/catalogs-contrib/catalog-jdbc-clickhouse/src/main/java/org/apache/gravitino/catalog/clickhouse/operations/ClickHouseTableSqlUtils.java b/catalogs-contrib/catalog-jdbc-clickhouse/src/main/java/org/apache/gravitino/catalog/clickhouse/operations/ClickHouseTableSqlUtils.java index 4f21d69cd3..afea01779f 100644 --- a/catalogs-contrib/catalog-jdbc-clickhouse/src/main/java/org/apache/gravitino/catalog/clickhouse/operations/ClickHouseTableSqlUtils.java +++ b/catalogs-contrib/catalog-jdbc-clickhouse/src/main/java/org/apache/gravitino/catalog/clickhouse/operations/ClickHouseTableSqlUtils.java @@ -68,17 +68,18 @@ final class ClickHouseTableSqlUtils { static String toPartitionExpression(Transform transform) { Preconditions.checkArgument(transform != null, "Partition transform cannot be null"); - Preconditions.checkArgument( - StringUtils.equalsIgnoreCase(transform.name(), Transforms.NAME_OF_IDENTITY), - "Unsupported partition transform: " + transform.name()); - Preconditions.checkArgument( - transform.arguments().length == 1 - && transform.arguments()[0] instanceof NamedReference - && ((NamedReference) transform.arguments()[0]).fieldName().length == 1, - "ClickHouse only supports single column identity partitioning"); - - String fieldName = ((NamedReference) transform.arguments()[0]).fieldName()[0]; - return quoteIdentifier(fieldName); + String fieldName = partitionFieldName(transform); + + if (StringUtils.equalsIgnoreCase(transform.name(), Transforms.NAME_OF_IDENTITY)) { + return quoteIdentifier(fieldName); + } else if (StringUtils.equalsIgnoreCase(transform.name(), Transforms.NAME_OF_YEAR)) { + return "toYear(%s)".formatted(quoteIdentifier(fieldName)); + } else if (StringUtils.equalsIgnoreCase(transform.name(), Transforms.NAME_OF_MONTH)) { + return "toYYYYMM(%s)".formatted(quoteIdentifier(fieldName)); + } else if (StringUtils.equalsIgnoreCase(transform.name(), Transforms.NAME_OF_DAY)) { + return "toDate(%s)".formatted(quoteIdentifier(fieldName)); + } + throw new IllegalArgumentException("Unsupported partition transform: " + transform.name()); } static List<String> extractShardingKeyColumns(String shardingKey) { @@ -236,4 +237,14 @@ final class ClickHouseTableSqlUtils { private static String quoteIdentifier(String identifier) { return String.format("`%s`", identifier); } + + private static String partitionFieldName(Transform transform) { + Preconditions.checkArgument( + transform.arguments().length == 1 + && transform.arguments()[0] instanceof NamedReference + && ((NamedReference) transform.arguments()[0]).fieldName().length == 1, + "ClickHouse only supports single column partitioning"); + + return ((NamedReference) transform.arguments()[0]).fieldName()[0]; + } } diff --git a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseIT.java b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseIT.java index ab3d95b870..df6de47525 100644 --- a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseIT.java +++ b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseIT.java @@ -497,7 +497,7 @@ public class CatalogClickHouseIT extends BaseIT { Column.of("amount", Types.FloatType.get(), "amt") }; - Transform[] partitioning = new Transform[] {Transforms.identity("event_time")}; + Transform[] partitioning = new Transform[] {Transforms.month("event_time")}; SortOrder[] sortOrders = new SortOrder[] { SortOrders.of(NamedReference.field("user_id"), SortDirection.ASCENDING), @@ -526,6 +526,7 @@ public class CatalogClickHouseIT extends BaseIT { Table loaded = catalog.asTableCatalog().loadTable(ident); Assertions.assertEquals(1, loaded.partitioning().length); + Assertions.assertEquals(Transforms.NAME_OF_MONTH, loaded.partitioning()[0].name()); Assertions.assertEquals( "event_time", ((NamedReference) loaded.partitioning()[0].arguments()[0]).fieldName()[0]); diff --git a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseTableOperations.java b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseTableOperations.java index 47188929e7..58a92a1052 100644 --- a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseTableOperations.java +++ b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseTableOperations.java @@ -909,6 +909,42 @@ public class TestClickHouseTableOperations extends TestClickHouse { ClickHouseUtils.getSortOrders("c1")); Assertions.assertTrue(partitionSql.contains("PARTITION BY `c1`")); + String yearPartitionSql = + ops.buildCreateSql( + "t_year", + new JdbcColumn[] {col}, + null, + propsWithPartition, + new Transform[] {Transforms.year("c1")}, + Distributions.NONE, + indexes, + ClickHouseUtils.getSortOrders("c1")); + Assertions.assertTrue(yearPartitionSql.contains("PARTITION BY toYear(`c1`)")); + + String monthPartitionSql = + ops.buildCreateSql( + "t_month", + new JdbcColumn[] {col}, + null, + propsWithPartition, + new Transform[] {Transforms.month("c1")}, + Distributions.NONE, + indexes, + ClickHouseUtils.getSortOrders("c1")); + Assertions.assertTrue(monthPartitionSql.contains("PARTITION BY toYYYYMM(`c1`)")); + + String dayPartitionSql = + ops.buildCreateSql( + "t_day", + new JdbcColumn[] {col}, + null, + propsWithPartition, + new Transform[] {Transforms.day("c1")}, + Distributions.NONE, + indexes, + ClickHouseUtils.getSortOrders("c1")); + Assertions.assertTrue(dayPartitionSql.contains("PARTITION BY toDate(`c1`)")); + // distribution not NONE Assertions.assertThrows( IllegalArgumentException.class,
