This is an automated email from the ASF dual-hosted git repository.
diqiu50 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 6402759db7 [#11017] improvement(clickhouse): Support date partition
transforms (#11018)
6402759db7 is described below
commit 6402759db7e85c285ba7c784b91ac703ca98f723
Author: Qi Yu <[email protected]>
AuthorDate: Fri May 15 10:36:19 2026 +0800
[#11017] improvement(clickhouse): Support date partition transforms (#11018)
### What changes were proposed in this pull request?
Support ClickHouse table creation with Gravitino date partition
transforms:
- `Transforms.year(column)` -> `PARTITION BY toYear(column)`
- `Transforms.month(column)` -> `PARTITION BY toYYYYMM(column)`
- `Transforms.day(column)` -> `PARTITION BY toDate(column)`
This PR also removes the stale identity-only helper in
`ClickHouseTableOperations` and uses the shared ClickHouse SQL utility
path for partition expression rendering.
### Why are the changes needed?
ClickHouse catalog table creation previously only supported identity
partitioning, while loading existing ClickHouse tables already
recognized `toYear`, `toYYYYMM`, and `toDate` partition expressions.
Fix: #11017
### Does this PR introduce _any_ user-facing change?
Yes. Users can now create ClickHouse MergeTree-family tables with
`year`, `month`, and `day` partition transforms through Gravitino.
### How was this patch tested?
```bash
./gradlew :catalogs-contrib:catalog-jdbc-clickhouse:spotlessApply
./gradlew :catalogs-contrib:catalog-jdbc-clickhouse:test --tests
org.apache.gravitino.catalog.clickhouse.operations.TestClickHouseTableOperations
-PskipITs
./gradlew :catalogs-contrib:catalog-jdbc-clickhouse:test --tests
org.apache.gravitino.catalog.clickhouse.integration.test.CatalogClickHouseIT.testCreateAndLoadWithPartitionSortAndIndexes
-PskipTests -PskipDockerTests=false
```
---------
Co-authored-by: Claude Opus 4.7 <[email protected]>
---
.../operations/ClickHouseTableOperations.java | 16 ------
.../operations/ClickHouseTableSqlUtils.java | 34 ++++++++----
.../integration/test/CatalogClickHouseIT.java | 46 +++++++++++++++-
.../operations/TestClickHouseTableOperations.java | 64 ++++++++++++++++++++++
4 files changed, 131 insertions(+), 29 deletions(-)
diff --git
a/catalogs-contrib/catalog-jdbc-clickhouse/src/main/java/org/apache/gravitino/catalog/clickhouse/operations/ClickHouseTableOperations.java
b/catalogs-contrib/catalog-jdbc-clickhouse/src/main/java/org/apache/gravitino/catalog/clickhouse/operations/ClickHouseTableOperations.java
index 4c453b7e7a..fc823ef78c 100644
---
a/catalogs-contrib/catalog-jdbc-clickhouse/src/main/java/org/apache/gravitino/catalog/clickhouse/operations/ClickHouseTableOperations.java
+++
b/catalogs-contrib/catalog-jdbc-clickhouse/src/main/java/org/apache/gravitino/catalog/clickhouse/operations/ClickHouseTableOperations.java
@@ -419,22 +419,6 @@ public class ClickHouseTableOperations extends
JdbcTableOperations {
.orElse(null);
}
- private String toPartitionExpression(Transform transform) {
- Preconditions.checkArgument(transform != null, "Partition transform cannot
be null");
- Preconditions.checkArgument(
- StringUtils.equalsIgnoreCase(transform.name(),
Transforms.NAME_OF_IDENTITY),
- "Unsupported partition transform: " + transform.name());
- Preconditions.checkArgument(
- transform.arguments().length == 1
- && transform.arguments()[0] instanceof NamedReference
- && ((NamedReference) transform.arguments()[0]).fieldName().length
== 1,
- "ClickHouse only supports single column identity partitioning");
-
- String fieldName =
- ((NamedReference) transform.arguments()[0]).fieldName()[0]; // already
validated
- return quoteIdentifier(fieldName);
- }
-
private void validateNoAutoIncrementColumns(JdbcColumn[] columns) {
if (ArrayUtils.isEmpty(columns)) {
return;
diff --git
a/catalogs-contrib/catalog-jdbc-clickhouse/src/main/java/org/apache/gravitino/catalog/clickhouse/operations/ClickHouseTableSqlUtils.java
b/catalogs-contrib/catalog-jdbc-clickhouse/src/main/java/org/apache/gravitino/catalog/clickhouse/operations/ClickHouseTableSqlUtils.java
index 4f21d69cd3..f90d607f23 100644
---
a/catalogs-contrib/catalog-jdbc-clickhouse/src/main/java/org/apache/gravitino/catalog/clickhouse/operations/ClickHouseTableSqlUtils.java
+++
b/catalogs-contrib/catalog-jdbc-clickhouse/src/main/java/org/apache/gravitino/catalog/clickhouse/operations/ClickHouseTableSqlUtils.java
@@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
@@ -68,17 +69,18 @@ final class ClickHouseTableSqlUtils {
static String toPartitionExpression(Transform transform) {
Preconditions.checkArgument(transform != null, "Partition transform cannot
be null");
- Preconditions.checkArgument(
- StringUtils.equalsIgnoreCase(transform.name(),
Transforms.NAME_OF_IDENTITY),
- "Unsupported partition transform: " + transform.name());
- Preconditions.checkArgument(
- transform.arguments().length == 1
- && transform.arguments()[0] instanceof NamedReference
- && ((NamedReference) transform.arguments()[0]).fieldName().length
== 1,
- "ClickHouse only supports single column identity partitioning");
-
- String fieldName = ((NamedReference)
transform.arguments()[0]).fieldName()[0];
- return quoteIdentifier(fieldName);
+ String name = transform.name().toLowerCase(Locale.ROOT);
+ return switch (name) {
+ case Transforms.NAME_OF_IDENTITY ->
quoteIdentifier(partitionFieldName(transform));
+ case Transforms.NAME_OF_YEAR -> "toYear(%s)"
+ .formatted(quoteIdentifier(partitionFieldName(transform)));
+ case Transforms.NAME_OF_MONTH -> "toYYYYMM(%s)"
+ .formatted(quoteIdentifier(partitionFieldName(transform)));
+ case Transforms.NAME_OF_DAY -> "toDate(%s)"
+ .formatted(quoteIdentifier(partitionFieldName(transform)));
+ default -> throw new IllegalArgumentException(
+ "Unsupported partition transform: " + transform.name());
+ };
}
static List<String> extractShardingKeyColumns(String shardingKey) {
@@ -236,4 +238,14 @@ final class ClickHouseTableSqlUtils {
private static String quoteIdentifier(String identifier) {
return String.format("`%s`", identifier);
}
+
+ private static String partitionFieldName(Transform transform) {
+ Preconditions.checkArgument(
+ transform.arguments().length == 1
+ && transform.arguments()[0] instanceof NamedReference
+ && ((NamedReference) transform.arguments()[0]).fieldName().length
== 1,
+ "ClickHouse partition transform only supports a single column
reference");
+
+ return ((NamedReference) transform.arguments()[0]).fieldName()[0];
+ }
}
diff --git
a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseIT.java
b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseIT.java
index ab3d95b870..eeb31d5731 100644
---
a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseIT.java
+++
b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseIT.java
@@ -485,7 +485,7 @@ public class CatalogClickHouseIT extends BaseIT {
NameIdentifier ident = NameIdentifier.of(schemaName, table);
Column[] cols =
new Column[] {
- Column.of("id", Types.LongType.get(), "id"),
+ Column.of("id", Types.LongType.get(), "id", false, false,
DEFAULT_VALUE_NOT_SET),
Column.of(
"event_time",
Types.TimestampType.withoutTimeZone(),
@@ -497,7 +497,7 @@ public class CatalogClickHouseIT extends BaseIT {
Column.of("amount", Types.FloatType.get(), "amt")
};
- Transform[] partitioning = new Transform[]
{Transforms.identity("event_time")};
+ Transform[] partitioning = new Transform[]
{Transforms.month("event_time")};
SortOrder[] sortOrders =
new SortOrder[] {
SortOrders.of(NamedReference.field("user_id"),
SortDirection.ASCENDING),
@@ -526,6 +526,7 @@ public class CatalogClickHouseIT extends BaseIT {
Table loaded = catalog.asTableCatalog().loadTable(ident);
Assertions.assertEquals(1, loaded.partitioning().length);
+ Assertions.assertEquals(Transforms.NAME_OF_MONTH,
loaded.partitioning()[0].name());
Assertions.assertEquals(
"event_time", ((NamedReference)
loaded.partitioning()[0].arguments()[0]).fieldName()[0]);
@@ -555,6 +556,47 @@ public class CatalogClickHouseIT extends BaseIT {
&& Arrays.deepEquals(idx.fieldNames(), new String[][]
{{"amount"}})));
}
+ @Test
+ void testCreateAndLoadWithPartitionTransforms() {
+ assertPartitionRoundTrip("identity_part",
Transforms.identity("event_time"));
+ assertPartitionRoundTrip("year_part", Transforms.year("event_time"));
+ assertPartitionRoundTrip("month_part", Transforms.month("event_time"));
+ assertPartitionRoundTrip("day_part", Transforms.day("event_time"));
+ }
+
+ private void assertPartitionRoundTrip(String prefix, Transform partition) {
+ String table = GravitinoITUtils.genRandomName(prefix);
+ NameIdentifier ident = NameIdentifier.of(schemaName, table);
+ Column[] cols =
+ new Column[] {
+ Column.of("id", Types.LongType.get(), "integer", false, false,
DEFAULT_VALUE_NOT_SET),
+ Column.of(
+ "event_time",
+ Types.TimestampType.withoutTimeZone(),
+ "ts",
+ false,
+ false,
+ DEFAULT_VALUE_NOT_SET)
+ };
+
+ catalog
+ .asTableCatalog()
+ .createTable(
+ ident,
+ cols,
+ "partition transform roundtrip",
+ createProperties(),
+ new Transform[] {partition},
+ Distributions.NONE,
+ getSortOrders("id"));
+
+ Table loaded = catalog.asTableCatalog().loadTable(ident);
+ Assertions.assertEquals(1, loaded.partitioning().length);
+ Assertions.assertEquals(partition.name(), loaded.partitioning()[0].name());
+ Assertions.assertEquals(
+ "event_time", ((NamedReference)
loaded.partitioning()[0].arguments()[0]).fieldName()[0]);
+ }
+
@Test
void testColumnNameWithKeyWords() {
// Create table from Gravitino API
diff --git
a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseTableOperations.java
b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseTableOperations.java
index 47188929e7..68f72d2e12 100644
---
a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseTableOperations.java
+++
b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseTableOperations.java
@@ -909,6 +909,70 @@ public class TestClickHouseTableOperations extends
TestClickHouse {
ClickHouseUtils.getSortOrders("c1"));
Assertions.assertTrue(partitionSql.contains("PARTITION BY `c1`"));
+ String yearPartitionSql =
+ ops.buildCreateSql(
+ "t_year",
+ new JdbcColumn[] {col},
+ null,
+ propsWithPartition,
+ new Transform[] {Transforms.year("c1")},
+ Distributions.NONE,
+ indexes,
+ ClickHouseUtils.getSortOrders("c1"));
+ Assertions.assertTrue(yearPartitionSql.contains("PARTITION BY
toYear(`c1`)"));
+
+ String monthPartitionSql =
+ ops.buildCreateSql(
+ "t_month",
+ new JdbcColumn[] {col},
+ null,
+ propsWithPartition,
+ new Transform[] {Transforms.month("c1")},
+ Distributions.NONE,
+ indexes,
+ ClickHouseUtils.getSortOrders("c1"));
+ Assertions.assertTrue(monthPartitionSql.contains("PARTITION BY
toYYYYMM(`c1`)"));
+
+ String dayPartitionSql =
+ ops.buildCreateSql(
+ "t_day",
+ new JdbcColumn[] {col},
+ null,
+ propsWithPartition,
+ new Transform[] {Transforms.day("c1")},
+ Distributions.NONE,
+ indexes,
+ ClickHouseUtils.getSortOrders("c1"));
+ Assertions.assertTrue(dayPartitionSql.contains("PARTITION BY
toDate(`c1`)"));
+
+ String multiPartitionSql =
+ ops.buildCreateSql(
+ "t_multi",
+ new JdbcColumn[] {col},
+ null,
+ propsWithPartition,
+ new Transform[] {Transforms.year("c1"), Transforms.day("c1")},
+ Distributions.NONE,
+ indexes,
+ ClickHouseUtils.getSortOrders("c1"));
+ Assertions.assertTrue(
+ multiPartitionSql.contains("PARTITION BY tuple(toYear(`c1`),
toDate(`c1`))"));
+
+ IllegalArgumentException exception =
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ ops.buildCreateSql(
+ "t_bucket",
+ new JdbcColumn[] {col},
+ null,
+ propsWithPartition,
+ new Transform[] {Transforms.bucket(8, new String[]
{"c1"})},
+ Distributions.NONE,
+ indexes,
+ ClickHouseUtils.getSortOrders("c1")));
+ Assertions.assertEquals("Unsupported partition transform: bucket",
exception.getMessage());
+
// distribution not NONE
Assertions.assertThrows(
IllegalArgumentException.class,
