This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 7592ddd [HUDI-2399] Rebalance CI jobs for shorter wait time (#3604)
7592ddd is described below
commit 7592ddd776148bf750043d31d74211670b961c46
Author: Raymond Xu <2701446+xushi...@users.noreply.github.com>
AuthorDate: Sun Sep 5 09:25:57 2021 -0700
[HUDI-2399] Rebalance CI jobs for shorter wait time (#3604)
---
azure-pipelines.yml | 133 +++++++++++++++++++++++++++++++++++++++-------------
1 file changed, 100 insertions(+), 33 deletions(-)
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index d290a7c..a0bf6a7 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -23,92 +23,158 @@ pool:
variables:
MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository
- MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) -Dcheckstyle.skip=true
-Drat.skip=true'
+ MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) -Dcheckstyle.skip=true
-Drat.skip=true -Djacoco.skip=true'
SPARK_VERSION: '2.4.4'
HADOOP_VERSION: '2.7'
SPARK_HOME:
$(Pipeline.Workspace)/spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)
stages:
- stage: test
- displayName: "Run tests"
jobs:
- - job: unit_tests_spark_client
+ - job: UT_FT_1
+ displayName: UT FT common & flink & UT client/spark-client
steps:
- task: Cache@2
+ displayName: set cache
inputs:
key: 'maven | "$(Agent.OS)" | **/pom.xml'
restoreKeys: |
maven | "$(Agent.OS)"
maven
path: $(MAVEN_CACHE_FOLDER)
- displayName: Cache Maven local repo
- - script: |
- mvn $(MAVEN_OPTS) clean install -DskipTests
- task: Maven@3
+ displayName: maven install
+ inputs:
+ mavenPomFile: 'pom.xml'
+ goals: 'install'
+ options: -DskipTests
+ publishJUnitResults: false
+ jdkVersionOption: '1.8'
+ mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
+ - task: Maven@3
+ displayName: UT common flink client/spark-client
+ inputs:
+ mavenPomFile: 'pom.xml'
+ goals: 'test'
+ options: -Punit-tests -pl
hudi-common,hudi-flink,hudi-client/hudi-spark-client
+ publishJUnitResults: false
+ jdkVersionOption: '1.8'
+ mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
+ - task: Maven@3
+ displayName: FT common flink
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
- options: -Punit-tests -pl hudi-client/hudi-spark-client
+ options: -Pfunctional-tests -pl hudi-common,hudi-flink
publishJUnitResults: false
- testResultsFiles: '**/surefire-reports/TEST-*.xml'
- testRunTitle: 'unit tests spark client'
- javaHomeOption: 'JDKVersion'
jdkVersionOption: '1.8'
- jdkArchitectureOption: 'x64'
mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
- - job: unit_tests_utilities
+ - job: UT_FT_2
+ displayName: FT client/spark-client
steps:
- task: Cache@2
+ displayName: set cache
inputs:
key: 'maven | "$(Agent.OS)" | **/pom.xml'
restoreKeys: |
maven | "$(Agent.OS)"
maven
path: $(MAVEN_CACHE_FOLDER)
- displayName: Cache Maven local repo
- - script: |
- mvn $(MAVEN_OPTS) clean install -DskipTests
- task: Maven@3
+ displayName: maven install
+ inputs:
+ mavenPomFile: 'pom.xml'
+ goals: 'install'
+ options: -DskipTests
+ publishJUnitResults: false
+ jdkVersionOption: '1.8'
+ mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
+ - task: Maven@3
+ displayName: FT client/spark-client
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
- options: -Punit-tests -pl hudi-utilities
+ options: -Pfunctional-tests -pl hudi-client/hudi-spark-client
publishJUnitResults: false
- testResultsFiles: '**/surefire-reports/TEST-*.xml'
- testRunTitle: 'unit tests utilities'
- javaHomeOption: 'JDKVersion'
jdkVersionOption: '1.8'
- jdkArchitectureOption: 'x64'
mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
- - job: unit_tests_other_modules
+ - job: UT_FT_3
+ displayName: UT FT cli & utilities & sync/hive-sync
steps:
- task: Cache@2
+ displayName: set cache
inputs:
key: 'maven | "$(Agent.OS)" | **/pom.xml'
restoreKeys: |
maven | "$(Agent.OS)"
maven
path: $(MAVEN_CACHE_FOLDER)
- displayName: Cache Maven local repo
- - script: |
- mvn $(MAVEN_OPTS) clean install -DskipTests
- task: Maven@3
+ displayName: maven install
+ inputs:
+ mavenPomFile: 'pom.xml'
+ goals: 'install'
+ options: -DskipTests
+ publishJUnitResults: false
+ jdkVersionOption: '1.8'
+ mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
+ - task: Maven@3
+ displayName: UT cli & utilities & sync/hive-sync
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
- options: -Punit-tests -pl
!hudi-utilities,!hudi-client/hudi-spark-client
+ options: -Punit-tests -pl
hudi-cli,hudi-utilities,hudi-sync/hudi-hive-sync
publishJUnitResults: false
- testResultsFiles: '**/surefire-reports/TEST-*.xml'
- testRunTitle: 'unit tests other modules'
- javaHomeOption: 'JDKVersion'
jdkVersionOption: '1.8'
- jdkArchitectureOption: 'x64'
mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
- - job: functional_tests
+ - task: Maven@3
+ displayName: FT cli & utilities & sync/hive-sync
+ inputs:
+ mavenPomFile: 'pom.xml'
+ goals: 'test'
+ options: -Pfunctional-tests -pl
hudi-cli,hudi-utilities,hudi-sync/hudi-hive-sync
+ publishJUnitResults: false
+ jdkVersionOption: '1.8'
+ mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
+ - job: UT_FT_4
+ displayName: UT FT other modules
steps:
- - script: |
- mvn $(MAVEN_OPTS) -Pfunctional-tests test
- - job: integration_tests
+ - task: Cache@2
+ displayName: set cache
+ inputs:
+ key: 'maven | "$(Agent.OS)" | **/pom.xml'
+ restoreKeys: |
+ maven | "$(Agent.OS)"
+ maven
+ path: $(MAVEN_CACHE_FOLDER)
+ - task: Maven@3
+ displayName: maven install
+ inputs:
+ mavenPomFile: 'pom.xml'
+ goals: 'install'
+ options: -DskipTests
+ publishJUnitResults: false
+ jdkVersionOption: '1.8'
+ mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
+ - task: Maven@3
+ displayName: UT other modules
+ inputs:
+ mavenPomFile: 'pom.xml'
+ goals: 'test'
+ options: -Punit-tests -pl
!hudi-common,!hudi-flink,!hudi-client/hudi-spark-client,!hudi-cli,!hudi-utilities,!hudi-sync/hudi-hive-sync
+ publishJUnitResults: false
+ jdkVersionOption: '1.8'
+ mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
+ - task: Maven@3
+ displayName: FT other modules
+ inputs:
+ mavenPomFile: 'pom.xml'
+ goals: 'test'
+ options: -Pfunctional-tests -pl
!hudi-common,!hudi-flink,!hudi-client/hudi-spark-client,!hudi-cli,!hudi-utilities,!hudi-sync/hudi-hive-sync
+ publishJUnitResults: false
+ jdkVersionOption: '1.8'
+ mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
+ - job: IT
steps:
- script: |
echo 'Downloading
spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)'
@@ -116,3 +182,4 @@ stages:
tar -xvf $(Pipeline.Workspace)/spark-$(SPARK_VERSION).tgz -C
$(Pipeline.Workspace)/
mkdir /tmp/spark-events/
mvn $(MAVEN_OPTS) -Pintegration-tests verify
+ displayName: IT
