This is an automated email from the ASF dual-hosted git repository. pwason pushed a commit to branch release-0.14.0 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 688d6c07a2110a2dba0286f8277cfa8cb4bdb881 Author: Lokesh Jain <lj...@apache.org> AuthorDate: Sat Sep 9 08:43:29 2023 +0530 [HUDI-6820] Fixing CI stability issues (#9661) - We face frequent flakiness around 2 modules (hudi-hadoop-mr and hudi-java-client). so, moving them out to github actions from azure CI. - Added explicit timeouts for few of deltastreamer continuous tests so that those fail instead of timing out. --------- Co-authored-by: sivabalan <n.siv...@gmail.com> --- .github/workflows/bot.yml | 32 ++++++++++++++++++++++ azure-pipelines-20230430.yml | 2 ++ .../deltastreamer/TestHoodieDeltaStreamer.java | 5 ++++ 3 files changed, 39 insertions(+) diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index 0811c828e49..acd51b8e123 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -112,6 +112,38 @@ jobs: run: mvn test -Pfunctional-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS + test-hudi-hadoop-mr-and-hudi-java-client: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - scalaProfile: "scala-2.12" + sparkProfile: "spark3.2" + flinkProfile: "flink1.17" + + steps: + - uses: actions/checkout@v3 + - name: Set up JDK 8 + uses: actions/setup-java@v3 + with: + java-version: '8' + distribution: 'adopt' + architecture: x64 + - name: Build Project + env: + SCALA_PROFILE: ${{ matrix.scalaProfile }} + SPARK_PROFILE: ${{ matrix.sparkProfile }} + FLINK_PROFILE: ${{ matrix.flinkProfile }} + run: + mvn clean install -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"FLINK_PROFILE" -DskipTests=true -Phudi-platform-service $MVN_ARGS + - name: UT - hudi-hadoop-mr and hudi-client/hudi-java-client + env: + SCALA_PROFILE: ${{ matrix.scalaProfile }} + SPARK_PROFILE: ${{ matrix.sparkProfile }} + FLINK_PROFILE: ${{ matrix.flinkProfile }} + run: + mvn test -Punit-tests -fae -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"FLINK_PROFILE" -pl hudi-hadoop-mr,hudi-client/hudi-java-client $MVN_ARGS + test-spark-java17: runs-on: ubuntu-latest strategy: diff --git a/azure-pipelines-20230430.yml b/azure-pipelines-20230430.yml index 2da5ab0d4f9..25a149b5cf4 100644 --- a/azure-pipelines-20230430.yml +++ b/azure-pipelines-20230430.yml @@ -53,6 +53,8 @@ parameters: - name: job4UTModules type: object default: + - '!hudi-hadoop-mr' + - '!hudi-client/hudi-java-client' - '!hudi-client/hudi-spark-client' - '!hudi-common' - '!hudi-examples' diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java index 6324fb83fc9..2a7db25647e 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java @@ -120,6 +120,7 @@ import org.apache.spark.sql.types.StructField; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.CsvSource; @@ -869,6 +870,7 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase { defaultSchemaProviderClassName = FilebasedSchemaProvider.class.getName(); } + @Timeout(600) @ParameterizedTest @EnumSource(value = HoodieRecordType.class, names = {"AVRO", "SPARK"}) public void testUpsertsCOWContinuousMode(HoodieRecordType recordType) throws Exception { @@ -892,12 +894,14 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase { UtilitiesTestBase.Helpers.deleteFileFromDfs(fs, tableBasePath); } + @Timeout(600) @ParameterizedTest @EnumSource(value = HoodieRecordType.class, names = {"AVRO"}) public void testUpsertsMORContinuousModeShutdownGracefully(HoodieRecordType recordType) throws Exception { testUpsertsContinuousMode(HoodieTableType.MERGE_ON_READ, "continuous_cow", true, recordType); } + @Timeout(600) @ParameterizedTest @EnumSource(value = HoodieRecordType.class, names = {"AVRO", "SPARK"}) public void testUpsertsMORContinuousMode(HoodieRecordType recordType) throws Exception { @@ -1404,6 +1408,7 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase { UtilitiesTestBase.Helpers.deleteFileFromDfs(fs, tableBasePath); } + @Timeout(600) @ParameterizedTest @EnumSource(value = HoodieRecordType.class, names = {"AVRO", "SPARK"}) public void testAsyncClusteringServiceWithCompaction(HoodieRecordType recordType) throws Exception {