This is an automated email from the ASF dual-hosted git repository.
yuxia pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluss.git
The following commit(s) were added to refs/heads/main by this push:
new 295c7b8a [lake/iceberg] Introduce fluss-lake-iceberg module and
IcebergLakeStorage (#1339)
295c7b8a is described below
commit 295c7b8ad9e2547cb0fb10f95ab1de54e61ccd4a
Author: cxxwang <[email protected]>
AuthorDate: Tue Jul 15 19:32:29 2025 +0800
[lake/iceberg] Introduce fluss-lake-iceberg module and IcebergLakeStorage
(#1339)
---
.github/workflows/stage.sh | 3 +-
fluss-dist/pom.xml | 7 ++
fluss-dist/src/main/assemblies/plugins.xml | 7 ++
fluss-lake/fluss-lake-iceberg/pom.xml | 88 ++++++++++++++++++++++
.../fluss/lake/iceberg/IcebergLakeStorage.java | 43 +++++++++++
.../lake/iceberg/IcebergLakeStoragePlugin.java | 38 ++++++++++
.../src/main/resources/META-INF/NOTICE | 9 +++
...libaba.fluss.lake.lakestorage.LakeStoragePlugin | 19 +++++
.../src/test/resources/log4j2-test.properties | 27 +++++++
.../org.junit.jupiter.api.extension.Extension | 19 +++++
fluss-lake/fluss-lake-paimon/pom.xml | 9 ---
fluss-lake/pom.xml | 47 ++++++++++++
fluss-test-coverage/pom.xml | 2 +
13 files changed, 308 insertions(+), 10 deletions(-)
diff --git a/.github/workflows/stage.sh b/.github/workflows/stage.sh
index 831a48d4..8d10d9b1 100755
--- a/.github/workflows/stage.sh
+++ b/.github/workflows/stage.sh
@@ -27,7 +27,8 @@ fluss-flink/fluss-flink-1.20,\
fluss-flink/fluss-flink-1.19,\
fluss-flink/fluss-flink-1.18,\
fluss-lake,\
-fluss-lake/fluss-lake-paimon
+fluss-lake/fluss-lake-paimon,\
+fluss-lake/fluss-lake-iceberg
"
function get_test_modules_for_stage() {
diff --git a/fluss-dist/pom.xml b/fluss-dist/pom.xml
index c9a710a0..43dd80f0 100644
--- a/fluss-dist/pom.xml
+++ b/fluss-dist/pom.xml
@@ -97,6 +97,13 @@
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>com.alibaba.fluss</groupId>
+ <artifactId>fluss-lake-iceberg</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-shaded-hadoop-2-uber</artifactId>
diff --git a/fluss-dist/src/main/assemblies/plugins.xml
b/fluss-dist/src/main/assemblies/plugins.xml
index b610e5e2..3bffaf44 100644
--- a/fluss-dist/src/main/assemblies/plugins.xml
+++ b/fluss-dist/src/main/assemblies/plugins.xml
@@ -101,6 +101,13 @@
<destName>fluss-lake-paimon-${project.version}.jar</destName>
<fileMode>0644</fileMode>
</file>
+
+ <file>
+
<source>../fluss-lake/fluss-lake-iceberg/target/fluss-lake-iceberg-${project.version}.jar</source>
+ <outputDirectory>plugins/iceberg/</outputDirectory>
+ <destName>fluss-lake-iceberg-${project.version}.jar</destName>
+ <fileMode>0644</fileMode>
+ </file>
</files>
</assembly>
\ No newline at end of file
diff --git a/fluss-lake/fluss-lake-iceberg/pom.xml
b/fluss-lake/fluss-lake-iceberg/pom.xml
new file mode 100644
index 00000000..4218a4b2
--- /dev/null
+++ b/fluss-lake/fluss-lake-iceberg/pom.xml
@@ -0,0 +1,88 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>com.alibaba.fluss</groupId>
+ <artifactId>fluss-lake</artifactId>
+ <version>0.8-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>fluss-lake-iceberg</artifactId>
+ <name>Fluss : Lake : Iceberg</name>
+
+ <packaging>jar</packaging>
+
+ <properties>
+ <iceberg.version>1.9.1</iceberg.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.iceberg</groupId>
+ <artifactId>iceberg-core</artifactId>
+ <version>${iceberg.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.alibaba.fluss</groupId>
+ <artifactId>fluss-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-fluss</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+
<include>org.apache.iceberg:iceberg-core</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>*</artifact>
+ <excludes>
+ <exclude>LICENSE</exclude>
+ <exclude>NOTICE</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+
+</project>
\ No newline at end of file
diff --git
a/fluss-lake/fluss-lake-iceberg/src/main/java/com/alibaba/fluss/lake/iceberg/IcebergLakeStorage.java
b/fluss-lake/fluss-lake-iceberg/src/main/java/com/alibaba/fluss/lake/iceberg/IcebergLakeStorage.java
new file mode 100644
index 00000000..932dd2b9
--- /dev/null
+++
b/fluss-lake/fluss-lake-iceberg/src/main/java/com/alibaba/fluss/lake/iceberg/IcebergLakeStorage.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.alibaba.fluss.lake.iceberg;
+
+import com.alibaba.fluss.config.Configuration;
+import com.alibaba.fluss.lake.lakestorage.LakeCatalog;
+import com.alibaba.fluss.lake.lakestorage.LakeStorage;
+import com.alibaba.fluss.lake.writer.LakeTieringFactory;
+
+/** Iceberg implementation of {@link LakeStorage}. */
+public class IcebergLakeStorage implements LakeStorage {
+
+ private final Configuration icebergConfig;
+
+ public IcebergLakeStorage(Configuration configuration) {
+ this.icebergConfig = configuration;
+ }
+
+ @Override
+ public LakeTieringFactory<?, ?> createLakeTieringFactory() {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ @Override
+ public LakeCatalog createLakeCatalog() {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+}
diff --git
a/fluss-lake/fluss-lake-iceberg/src/main/java/com/alibaba/fluss/lake/iceberg/IcebergLakeStoragePlugin.java
b/fluss-lake/fluss-lake-iceberg/src/main/java/com/alibaba/fluss/lake/iceberg/IcebergLakeStoragePlugin.java
new file mode 100644
index 00000000..8c5ece2b
--- /dev/null
+++
b/fluss-lake/fluss-lake-iceberg/src/main/java/com/alibaba/fluss/lake/iceberg/IcebergLakeStoragePlugin.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.alibaba.fluss.lake.iceberg;
+
+import com.alibaba.fluss.config.Configuration;
+import com.alibaba.fluss.lake.lakestorage.LakeStorage;
+import com.alibaba.fluss.lake.lakestorage.LakeStoragePlugin;
+
+/** Iceberg implementation of {@link LakeStoragePlugin}. */
+public class IcebergLakeStoragePlugin implements LakeStoragePlugin {
+
+ private static final String IDENTIFIER = "iceberg";
+
+ @Override
+ public String identifier() {
+ return IDENTIFIER;
+ }
+
+ @Override
+ public LakeStorage createLakeStorage(Configuration configuration) {
+ return new IcebergLakeStorage(configuration);
+ }
+}
diff --git a/fluss-lake/fluss-lake-iceberg/src/main/resources/META-INF/NOTICE
b/fluss-lake/fluss-lake-iceberg/src/main/resources/META-INF/NOTICE
new file mode 100644
index 00000000..71f03774
--- /dev/null
+++ b/fluss-lake/fluss-lake-iceberg/src/main/resources/META-INF/NOTICE
@@ -0,0 +1,9 @@
+fluss-lake-iceberg
+Copyright 2025 The Apache Software Foundation
+
+This project includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+This project bundles the following dependencies under the Apache Software
License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt)
+
+- org.apache.iceberg:iceberg-core:1.9.1
diff --git
a/fluss-lake/fluss-lake-iceberg/src/main/resources/META-INF/services/com.alibaba.fluss.lake.lakestorage.LakeStoragePlugin
b/fluss-lake/fluss-lake-iceberg/src/main/resources/META-INF/services/com.alibaba.fluss.lake.lakestorage.LakeStoragePlugin
new file mode 100644
index 00000000..164379df
--- /dev/null
+++
b/fluss-lake/fluss-lake-iceberg/src/main/resources/META-INF/services/com.alibaba.fluss.lake.lakestorage.LakeStoragePlugin
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+com.alibaba.fluss.lake.iceberg.IcebergLakeStoragePlugin
\ No newline at end of file
diff --git
a/fluss-lake/fluss-lake-iceberg/src/test/resources/log4j2-test.properties
b/fluss-lake/fluss-lake-iceberg/src/test/resources/log4j2-test.properties
new file mode 100644
index 00000000..fabc6bf5
--- /dev/null
+++ b/fluss-lake/fluss-lake-iceberg/src/test/resources/log4j2-test.properties
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Set root logger level to OFF to not flood build logs
+# set manually to INFO for debugging purposes
+rootLogger.level=OFF
+rootLogger.appenderRef.test.ref=TestLogger
+appender.testlogger.name=TestLogger
+appender.testlogger.type=CONSOLE
+appender.testlogger.target=SYSTEM_ERR
+appender.testlogger.layout.type=PatternLayout
+appender.testlogger.layout.pattern=%-4r [%t] %-5p %c %x - %m%n
+
diff --git
a/fluss-lake/fluss-lake-iceberg/src/test/resources/org.junit.jupiter.api.extension.Extension
b/fluss-lake/fluss-lake-iceberg/src/test/resources/org.junit.jupiter.api.extension.Extension
new file mode 100644
index 00000000..590243ea
--- /dev/null
+++
b/fluss-lake/fluss-lake-iceberg/src/test/resources/org.junit.jupiter.api.extension.Extension
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+com.alibaba.fluss.testutils.common.TestLoggerExtension
\ No newline at end of file
diff --git a/fluss-lake/fluss-lake-paimon/pom.xml
b/fluss-lake/fluss-lake-paimon/pom.xml
index 13347296..322caa99 100644
--- a/fluss-lake/fluss-lake-paimon/pom.xml
+++ b/fluss-lake/fluss-lake-paimon/pom.xml
@@ -34,8 +34,6 @@
<properties>
<paimon.version>1.0.1</paimon.version>
- <flink.major.version>1.20</flink.major.version>
- <flink.version>1.20.1</flink.version>
</properties>
<dependencies>
@@ -135,7 +133,6 @@
<dependency>
<groupId>com.alibaba.fluss</groupId>
<artifactId>fluss-flink-${flink.major.version}</artifactId>
- <version>${project.version}</version>
<scope>test</scope>
</dependency>
@@ -164,42 +161,36 @@
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
- <version>${flink.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-common</artifactId>
- <version>${flink.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-runtime</artifactId>
- <version>${flink.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-base</artifactId>
- <version>${flink.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-files</artifactId>
- <version>${flink.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-test-utils</artifactId>
- <version>${flink.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
diff --git a/fluss-lake/pom.xml b/fluss-lake/pom.xml
index cdf37de6..dadae6d5 100644
--- a/fluss-lake/pom.xml
+++ b/fluss-lake/pom.xml
@@ -27,10 +27,57 @@
<version>0.8-SNAPSHOT</version>
</parent>
+ <properties>
+ <flink.major.version>1.20</flink.major.version>
+ <flink.version>1.20.1</flink.version>
+ </properties>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${flink.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-common</artifactId>
+ <version>${flink.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-runtime</artifactId>
+ <version>${flink.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-base</artifactId>
+ <version>${flink.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-files</artifactId>
+ <version>${flink.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-test-utils</artifactId>
+ <version>${flink.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.alibaba.fluss</groupId>
+ <artifactId>fluss-flink-${flink.major.version}</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+
<artifactId>fluss-lake</artifactId>
<name>Fluss : Lake :</name>
<modules>
<module>fluss-lake-paimon</module>
+ <module>fluss-lake-iceberg</module>
</modules>
<packaging>pom</packaging>
</project>
\ No newline at end of file
diff --git a/fluss-test-coverage/pom.xml b/fluss-test-coverage/pom.xml
index 20d8d2c8..a661722e 100644
--- a/fluss-test-coverage/pom.xml
+++ b/fluss-test-coverage/pom.xml
@@ -325,6 +325,8 @@
<exclude>com.alibaba.fluss.lake.batch.ArrowRecordBatch</exclude>
<exclude>com.alibaba.fluss.lake.committer.CommittedLakeSnapshot</exclude>
<exclude>com.alibaba.fluss.lake.paimon.FlussDataTypeToPaimonDataType</exclude>
+ <!-- temporarily exclude iceberg -->
+
<exclude>com.alibaba.fluss.lake.iceberg.*</exclude>
<!-- start exclude for flink tiering
service -->
<exclude>com.alibaba.fluss.flink.tiering.source.TieringSourceOptions</exclude>
<exclude>com.alibaba.fluss.flink.tiering.source.TieringSource.Builder</exclude>
