This is an automated email from the ASF dual-hosted git repository.
zouxxyy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 036f5c598f [spark] support using scala 2.13 to build and deploy with
paimon-spark (#6128)
036f5c598f is described below
commit 036f5c598facfc2255fcb0c0c316cfe9fcc656e9
Author: Houliang Qi <[email protected]>
AuthorDate: Thu Sep 18 00:53:37 2025 +0800
[spark] support using scala 2.13 to build and deploy with paimon-spark
(#6128)
Co-authored-by: zouxxyy <[email protected]>
---
.github/workflows/publish_snapshot-jdk17.yml | 4 ++--
.github/workflows/publish_snapshot.yml | 2 ++
.github/workflows/utitcase-jdk11.yml | 4 ++--
.github/workflows/utitcase-spark-3.x.yml | 20 +++++++++++++-------
.github/workflows/utitcase-spark-4.x.yml | 7 +++++--
.github/workflows/utitcase.yml | 4 ++--
paimon-spark/paimon-spark-3.2/pom.xml | 10 +++++-----
paimon-spark/paimon-spark-3.3/pom.xml | 10 +++++-----
paimon-spark/paimon-spark-3.4/pom.xml | 10 +++++-----
paimon-spark/paimon-spark-3.5/pom.xml | 10 +++++-----
paimon-spark/paimon-spark-4.0/pom.xml | 10 +++++-----
paimon-spark/paimon-spark-ut/pom.xml | 4 ++--
paimon-spark/paimon-spark3-common/pom.xml | 4 ++--
paimon-spark/paimon-spark4-common/pom.xml | 4 ++--
pom.xml | 19 ++++++++++++++++---
15 files changed, 73 insertions(+), 49 deletions(-)
diff --git a/.github/workflows/publish_snapshot-jdk17.yml
b/.github/workflows/publish_snapshot-jdk17.yml
index b1542625b7..177a0f850e 100644
--- a/.github/workflows/publish_snapshot-jdk17.yml
+++ b/.github/workflows/publish_snapshot-jdk17.yml
@@ -63,7 +63,7 @@ jobs:
echo "<password>$ASF_PASSWORD</password>" >> $tmp_settings
echo "</server></servers></settings>" >> $tmp_settings
- mvn --settings $tmp_settings clean install -Dgpg.skip -Drat.skip
-DskipTests -Papache-release,spark4,flink1 -pl
org.apache.paimon:paimon-spark-4.0 -am
- mvn --settings $tmp_settings clean deploy -Dgpg.skip -Drat.skip
-DskipTests -Papache-release,spark4,flink1 -pl
org.apache.paimon:paimon-spark-common_2.13,org.apache.paimon:paimon-spark4-common,org.apache.paimon:paimon-spark-4.0
+ mvn --settings $tmp_settings clean install -Dgpg.skip -Drat.skip
-DskipTests -Papache-release,spark4,flink1 -pl
org.apache.paimon:paimon-spark-4.0_2.13 -am
+ mvn --settings $tmp_settings clean deploy -Dgpg.skip -Drat.skip
-DskipTests -Papache-release,spark4,flink1 -pl
org.apache.paimon:paimon-spark-common_2.13,org.apache.paimon:paimon-spark4-common_2.13,org.apache.paimon:paimon-spark-ut_2.13,org.apache.paimon:paimon-spark-4.0_2.13
rm $tmp_settings
diff --git a/.github/workflows/publish_snapshot.yml
b/.github/workflows/publish_snapshot.yml
index a242cc65de..b9deaeb0e0 100644
--- a/.github/workflows/publish_snapshot.yml
+++ b/.github/workflows/publish_snapshot.yml
@@ -66,5 +66,7 @@ jobs:
echo "</server></servers></settings>" >> $tmp_settings
mvn --settings $tmp_settings clean deploy -Dgpg.skip -Drat.skip
-DskipTests -Papache-release,spark3,flink1
+ # deploy for scala 2.13
+ mvn --settings $tmp_settings clean deploy -Dgpg.skip -Drat.skip
-DskipTests -Papache-release,spark3,scala-2.13,flink1 -pl
org.apache.paimon:paimon-spark-common_2.13,org.apache.paimon:paimon-spark3-common_2.13,org.apache.paimon:paimon-spark-ut_2.13,org.apache.paimon:paimon-spark-3.2_2.13,org.apache.paimon:paimon-spark-3.3_2.13,org.apache.paimon:paimon-spark-3.4_2.13,org.apache.paimon:paimon-spark-3.5_2.13
rm $tmp_settings
diff --git a/.github/workflows/utitcase-jdk11.yml
b/.github/workflows/utitcase-jdk11.yml
index f2e538fe89..0386a02e14 100644
--- a/.github/workflows/utitcase-jdk11.yml
+++ b/.github/workflows/utitcase-jdk11.yml
@@ -52,8 +52,8 @@ jobs:
jvm_timezone=$(random_timezone)
echo "JVM timezone is set to $jvm_timezone"
test_modules="!paimon-e2e-tests,!org.apache.paimon:paimon-hive-connector-3.1,"
- for suffix in 3.5 3.4 3.3 3.2 ut; do
- test_modules+="!org.apache.paimon:paimon-spark-${suffix},"
+ for suffix in ut 3.5 3.4 3.3 3.2; do
+ test_modules+="!org.apache.paimon:paimon-spark-${suffix}_2.12,"
done
test_modules="${test_modules%,}"
mvn -T 1C -B clean install -pl "${test_modules}" -Pflink1,spark3
-Pskip-paimon-flink-tests -Duser.timezone=$jvm_timezone
diff --git a/.github/workflows/utitcase-spark-3.x.yml
b/.github/workflows/utitcase-spark-3.x.yml
index 5b17dc6e75..40ae73fab4 100644
--- a/.github/workflows/utitcase-spark-3.x.yml
+++ b/.github/workflows/utitcase-spark-3.x.yml
@@ -36,20 +36,26 @@ concurrency:
cancel-in-progress: true
jobs:
- build:
+ build_test:
runs-on: ubuntu-latest
-
+ strategy:
+ fail-fast: true
+ matrix:
+ scala_version: [ '2.12', '2.13' ]
steps:
- name: Checkout code
uses: actions/checkout@v4
+
- name: Set up JDK ${{ env.JDK_VERSION }}
uses: actions/setup-java@v4
with:
java-version: ${{ env.JDK_VERSION }}
distribution: 'temurin'
- - name: Build Spark
- run: mvn -T 2C -B clean install -DskipTests
- - name: Test Spark
+
+ - name: Build Spark 3 with ${{ matrix.scala_version }}
+ run: mvn -T 2C -B clean install -DskipTests -Pspark3,flink1,scala-${{
matrix.scala_version }}
+
+ - name: Test Spark 3 with ${{ matrix.scala_version }}
timeout-minutes: 60
run: |
# run tests with random timezone to find out timezone related bugs
@@ -58,9 +64,9 @@ jobs:
echo "JVM timezone is set to $jvm_timezone"
test_modules=""
for suffix in ut 3.5 3.4 3.3 3.2; do
- test_modules+="org.apache.paimon:paimon-spark-${suffix},"
+ test_modules+="org.apache.paimon:paimon-spark-${suffix}_${{
matrix.scala_version }},"
done
test_modules="${test_modules%,}"
- mvn -T 2C -B verify -pl "${test_modules}"
-Duser.timezone=$jvm_timezone
+ mvn -T 2C -B verify -pl "${test_modules}"
-Duser.timezone=$jvm_timezone -Pspark3,flink1,scala-${{ matrix.scala_version }}
env:
MAVEN_OPTS: -Xmx4096m
\ No newline at end of file
diff --git a/.github/workflows/utitcase-spark-4.x.yml
b/.github/workflows/utitcase-spark-4.x.yml
index 605d7ded84..b9b3071de5 100644
--- a/.github/workflows/utitcase-spark-4.x.yml
+++ b/.github/workflows/utitcase-spark-4.x.yml
@@ -36,19 +36,22 @@ concurrency:
cancel-in-progress: true
jobs:
- build:
+ build_test:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
+
- name: Set up JDK ${{ env.JDK_VERSION }}
uses: actions/setup-java@v4
with:
java-version: ${{ env.JDK_VERSION }}
distribution: 'temurin'
+
- name: Build Spark
run: mvn -T 2C -B clean install -DskipTests -Pspark4,flink1
+
- name: Test Spark
timeout-minutes: 60
run: |
@@ -58,7 +61,7 @@ jobs:
echo "JVM timezone is set to $jvm_timezone"
test_modules=""
for suffix in ut 4.0; do
- test_modules+="org.apache.paimon:paimon-spark-${suffix},"
+ test_modules+="org.apache.paimon:paimon-spark-${suffix}_2.13,"
done
test_modules="${test_modules%,}"
mvn -T 2C -B verify -pl "${test_modules}"
-Duser.timezone=$jvm_timezone -Pspark4,flink1
diff --git a/.github/workflows/utitcase.yml b/.github/workflows/utitcase.yml
index f16f5bf32a..3ba3df80fd 100644
--- a/.github/workflows/utitcase.yml
+++ b/.github/workflows/utitcase.yml
@@ -62,8 +62,8 @@ jobs:
echo "JVM timezone is set to $jvm_timezone"
TEST_MODULES="!paimon-e2e-tests,"
- for suffix in 3.5 3.4 3.3 3.2 ut; do
- TEST_MODULES+="!org.apache.paimon:paimon-spark-${suffix},"
+ for suffix in ut 3.5 3.4 3.3 3.2; do
+ TEST_MODULES+="!org.apache.paimon:paimon-spark-${suffix}_2.12,"
done
TEST_MODULES="${TEST_MODULES%,}"
mvn -T 2C -B clean install -pl "${TEST_MODULES}"
-Pskip-paimon-flink-tests -Duser.timezone=$jvm_timezone
diff --git a/paimon-spark/paimon-spark-3.2/pom.xml
b/paimon-spark/paimon-spark-3.2/pom.xml
index 2487548c29..1a98ff45d3 100644
--- a/paimon-spark/paimon-spark-3.2/pom.xml
+++ b/paimon-spark/paimon-spark-3.2/pom.xml
@@ -28,8 +28,8 @@ under the License.
<version>1.4-SNAPSHOT</version>
</parent>
- <artifactId>paimon-spark-3.2</artifactId>
- <name>Paimon : Spark : 3.2</name>
+ <artifactId>paimon-spark-3.2_${scala.binary.version}</artifactId>
+ <name>Paimon : Spark : 3.2 : ${scala.binary.version}</name>
<properties>
<spark.version>3.2.4</spark.version>
@@ -43,7 +43,7 @@ under the License.
<dependency>
<groupId>org.apache.paimon</groupId>
- <artifactId>paimon-spark3-common</artifactId>
+
<artifactId>paimon-spark3-common_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
@@ -75,7 +75,7 @@ under the License.
<dependency>
<groupId>org.apache.paimon</groupId>
- <artifactId>paimon-spark-ut</artifactId>
+ <artifactId>paimon-spark-ut_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
@@ -131,7 +131,7 @@ under the License.
</filters>
<artifactSet>
<includes combine.children="append">
-
<include>org.apache.paimon:paimon-spark3-common</include>
+
<include>org.apache.paimon:paimon-spark3-common_${scala.binary.version}</include>
</includes>
</artifactSet>
</configuration>
diff --git a/paimon-spark/paimon-spark-3.3/pom.xml
b/paimon-spark/paimon-spark-3.3/pom.xml
index 3b31d0105a..5b4396de66 100644
--- a/paimon-spark/paimon-spark-3.3/pom.xml
+++ b/paimon-spark/paimon-spark-3.3/pom.xml
@@ -28,8 +28,8 @@ under the License.
<version>1.4-SNAPSHOT</version>
</parent>
- <artifactId>paimon-spark-3.3</artifactId>
- <name>Paimon : Spark : 3.3</name>
+ <artifactId>paimon-spark-3.3_${scala.binary.version}</artifactId>
+ <name>Paimon : Spark : 3.3: ${scala.binary.version}</name>
<properties>
<spark.version>3.3.4</spark.version>
@@ -43,7 +43,7 @@ under the License.
<dependency>
<groupId>org.apache.paimon</groupId>
- <artifactId>paimon-spark3-common</artifactId>
+
<artifactId>paimon-spark3-common_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
@@ -75,7 +75,7 @@ under the License.
<dependency>
<groupId>org.apache.paimon</groupId>
- <artifactId>paimon-spark-ut</artifactId>
+ <artifactId>paimon-spark-ut_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
@@ -131,7 +131,7 @@ under the License.
</filters>
<artifactSet>
<includes combine.children="append">
-
<include>org.apache.paimon:paimon-spark3-common</include>
+
<include>org.apache.paimon:paimon-spark3-common_${scala.binary.version}</include>
</includes>
</artifactSet>
</configuration>
diff --git a/paimon-spark/paimon-spark-3.4/pom.xml
b/paimon-spark/paimon-spark-3.4/pom.xml
index 9e09f06d17..9b940591ad 100644
--- a/paimon-spark/paimon-spark-3.4/pom.xml
+++ b/paimon-spark/paimon-spark-3.4/pom.xml
@@ -28,8 +28,8 @@ under the License.
<version>1.4-SNAPSHOT</version>
</parent>
- <artifactId>paimon-spark-3.4</artifactId>
- <name>Paimon : Spark : 3.4</name>
+ <artifactId>paimon-spark-3.4_${scala.binary.version}</artifactId>
+ <name>Paimon : Spark : 3.4: ${scala.binary.version}</name>
<properties>
<spark.version>3.4.3</spark.version>
@@ -43,7 +43,7 @@ under the License.
<dependency>
<groupId>org.apache.paimon</groupId>
- <artifactId>paimon-spark3-common</artifactId>
+
<artifactId>paimon-spark3-common_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
@@ -75,7 +75,7 @@ under the License.
<dependency>
<groupId>org.apache.paimon</groupId>
- <artifactId>paimon-spark-ut</artifactId>
+ <artifactId>paimon-spark-ut_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
@@ -131,7 +131,7 @@ under the License.
</filters>
<artifactSet>
<includes combine.children="append">
-
<include>org.apache.paimon:paimon-spark3-common</include>
+
<include>org.apache.paimon:paimon-spark3-common_${scala.binary.version}</include>
</includes>
</artifactSet>
</configuration>
diff --git a/paimon-spark/paimon-spark-3.5/pom.xml
b/paimon-spark/paimon-spark-3.5/pom.xml
index 53557de64f..a8986a0579 100644
--- a/paimon-spark/paimon-spark-3.5/pom.xml
+++ b/paimon-spark/paimon-spark-3.5/pom.xml
@@ -28,8 +28,8 @@ under the License.
<version>1.4-SNAPSHOT</version>
</parent>
- <artifactId>paimon-spark-3.5</artifactId>
- <name>Paimon : Spark : 3.5</name>
+ <artifactId>paimon-spark-3.5_${scala.binary.version}</artifactId>
+ <name>Paimon : Spark : 3.5: ${scala.binary.version}</name>
<properties>
<spark.version>3.5.6</spark.version>
@@ -43,7 +43,7 @@ under the License.
<dependency>
<groupId>org.apache.paimon</groupId>
- <artifactId>paimon-spark3-common</artifactId>
+
<artifactId>paimon-spark3-common_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
@@ -75,7 +75,7 @@ under the License.
<dependency>
<groupId>org.apache.paimon</groupId>
- <artifactId>paimon-spark-ut</artifactId>
+ <artifactId>paimon-spark-ut_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
@@ -131,7 +131,7 @@ under the License.
</filters>
<artifactSet>
<includes combine.children="append">
-
<include>org.apache.paimon:paimon-spark3-common</include>
+
<include>org.apache.paimon:paimon-spark3-common_${scala.binary.version}</include>
</includes>
</artifactSet>
</configuration>
diff --git a/paimon-spark/paimon-spark-4.0/pom.xml
b/paimon-spark/paimon-spark-4.0/pom.xml
index b570810b99..27b8b2439d 100644
--- a/paimon-spark/paimon-spark-4.0/pom.xml
+++ b/paimon-spark/paimon-spark-4.0/pom.xml
@@ -28,8 +28,8 @@ under the License.
<version>1.4-SNAPSHOT</version>
</parent>
- <artifactId>paimon-spark-4.0</artifactId>
- <name>Paimon : Spark : 4.0</name>
+ <artifactId>paimon-spark-4.0_2.13</artifactId>
+ <name>Paimon : Spark : 4.0 : 2.13</name>
<properties>
<spark.version>4.0.1</spark.version>
@@ -43,7 +43,7 @@ under the License.
<dependency>
<groupId>org.apache.paimon</groupId>
- <artifactId>paimon-spark4-common</artifactId>
+
<artifactId>paimon-spark4-common_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
@@ -81,7 +81,7 @@ under the License.
<dependency>
<groupId>org.apache.paimon</groupId>
- <artifactId>paimon-spark-ut</artifactId>
+ <artifactId>paimon-spark-ut_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
@@ -143,7 +143,7 @@ under the License.
</filters>
<artifactSet>
<includes combine.children="append">
-
<include>org.apache.paimon:paimon-spark4-common</include>
+
<include>org.apache.paimon:paimon-spark4-common_${scala.binary.version}</include>
</includes>
</artifactSet>
</configuration>
diff --git a/paimon-spark/paimon-spark-ut/pom.xml
b/paimon-spark/paimon-spark-ut/pom.xml
index 10ad2317ef..e6eab99f17 100644
--- a/paimon-spark/paimon-spark-ut/pom.xml
+++ b/paimon-spark/paimon-spark-ut/pom.xml
@@ -28,8 +28,8 @@ under the License.
<version>1.4-SNAPSHOT</version>
</parent>
- <artifactId>paimon-spark-ut</artifactId>
- <name>Paimon : Spark : UT</name>
+ <artifactId>paimon-spark-ut_${scala.binary.version}</artifactId>
+ <name>Paimon : Spark : UT : ${scala.binary.version}</name>
<properties>
<spark.version>${paimon-spark-common.spark.version}</spark.version>
diff --git a/paimon-spark/paimon-spark3-common/pom.xml
b/paimon-spark/paimon-spark3-common/pom.xml
index b3a2172b8d..b782d1c137 100644
--- a/paimon-spark/paimon-spark3-common/pom.xml
+++ b/paimon-spark/paimon-spark3-common/pom.xml
@@ -30,8 +30,8 @@ under the License.
<packaging>jar</packaging>
- <artifactId>paimon-spark3-common</artifactId>
- <name>Paimon : Spark3 : Common</name>
+ <artifactId>paimon-spark3-common_${scala.binary.version}</artifactId>
+ <name>Paimon : Spark3 : Common : ${scala.binary.version}</name>
<properties>
<spark.version>${paimon-spark-common.spark.version}</spark.version>
diff --git a/paimon-spark/paimon-spark4-common/pom.xml
b/paimon-spark/paimon-spark4-common/pom.xml
index 7ce29fba1d..c8833fe141 100644
--- a/paimon-spark/paimon-spark4-common/pom.xml
+++ b/paimon-spark/paimon-spark4-common/pom.xml
@@ -30,8 +30,8 @@ under the License.
<packaging>jar</packaging>
- <artifactId>paimon-spark4-common</artifactId>
- <name>Paimon : Spark4 : Common</name>
+ <artifactId>paimon-spark4-common_2.13</artifactId>
+ <name>Paimon : Spark4 : Common : 2.13</name>
<properties>
<spark.version>${paimon-spark-common.spark.version}</spark.version>
diff --git a/pom.xml b/pom.xml
index 10153f26fc..ef8fa91300 100644
--- a/pom.xml
+++ b/pom.xml
@@ -126,7 +126,7 @@ under the License.
<test.mysql.connector.java.version>8.0.27</test.mysql.connector.java.version>
<!-- spark profile properties-->
- <paimon-sparkx-common>paimon-spark3-common</paimon-sparkx-common>
+
<paimon-sparkx-common>paimon-spark3-common_${scala.binary.version}</paimon-sparkx-common>
<paimon-spark-common.spark.version>3.5.6</paimon-spark-common.spark.version>
<test.spark.main.version>3.3</test.spark.main.version>
<test.spark.version>3.3.0</test.spark.version>
@@ -402,7 +402,7 @@ under the License.
<scala.binary.version>2.12</scala.binary.version>
<scala.version>${scala212.version}</scala.version>
<paimon-spark-common.spark.version>3.5.6</paimon-spark-common.spark.version>
-
<paimon-sparkx-common>paimon-spark3-common</paimon-sparkx-common>
+
<paimon-sparkx-common>paimon-spark3-common_${scala.binary.version}</paimon-sparkx-common>
<!-- todo: support the latest spark in paimon e2e test2 -->
<test.spark.main.version>3.3</test.spark.main.version>
<test.spark.version>3.3.0</test.spark.version>
@@ -427,7 +427,7 @@ under the License.
<scala.binary.version>2.13</scala.binary.version>
<scala.version>${scala213.version}</scala.version>
<paimon-spark-common.spark.version>4.0.1</paimon-spark-common.spark.version>
-
<paimon-sparkx-common>paimon-spark4-common</paimon-sparkx-common>
+
<paimon-sparkx-common>paimon-spark4-common_2.13</paimon-sparkx-common>
<test.spark.main.version>4.0</test.spark.main.version>
<test.spark.version>4.0.1</test.spark.version>
</properties>
@@ -483,6 +483,19 @@ under the License.
</activation>
</profile>
+ <profile>
+ <id>scala-2.13</id>
+ <properties>
+ <scala.binary.version>2.13</scala.binary.version>
+ <scala.version>${scala213.version}</scala.version>
+ </properties>
+ <activation>
+ <property>
+ <name>scala-2.13</name>
+ </property>
+ </activation>
+ </profile>
+
<profile>
<id>paimon-iceberg</id>
<modules>
