This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 3eb1fa9 [KYUUBI #2175] Improve CI with cancel & concurrency & paths
filter
3eb1fa9 is described below
commit 3eb1fa9e48b4c533f05b3719486df198c6e29751
Author: Kent Yao <[email protected]>
AuthorDate: Sat Mar 19 21:06:08 2022 +0800
[KYUUBI #2175] Improve CI with cancel & concurrency & paths filter
### _Why are the changes needed?_
Better ASF GHA resource usage, see more
https://cwiki.apache.org/confluence/display/BUILDS/GitHub+Actions+status
- cancel-in-progress enabled. w/ this, a previous build can be canceled if
new commit(s) have arrived
- style check w/o install. currently, the style check takes 8~10min to
finish because an install step before it. now, it shall return within a minute.
note that, this might 'fail' itself when we introduce a new module and others
depend on it, but it's ok to verify by the log whether a PR ok to be merged.
- dependency check runs only when pom.xml updates. this flow now runs only
pom changes as it is the only way that may cause dependency changes.
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [x] Add screenshots for manual tests if appropriate
**Canceled**:
https://github.com/apache/incubator-kyuubi/runs/5604809742?check_suite_focus=true
<img width="548" alt="image"
src="https://user-images.githubusercontent.com/8326978/159059722-480f4fd6-d87e-4b07-984e-eb5b08a84beb.png";>
- [x] [Run
test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests)
locally before make a pull request
Closes #2175 from yaooqinn/gha.
Closes #2175
33777bc5 [Kent Yao] Update .github/workflows/style.yml
ec41ddd0 [Kent Yao] Update .github/workflows/style.yml
79db981f [Kent Yao] fix
4e1fb387 [Kent Yao] fix
733df9cb [Kent Yao] Improve CI with cancel & concurrency
494c3a84 [Kent Yao] Revert "Improve CI with cancel & concurrency"
57641ac4 [Kent Yao] Improve CI with cancel & concurrency
6dafe970 [Kent Yao] Improve CI with cancel & concurrency
40160ab9 [Kent Yao] Improve CI with cancel & concurrency
377285c2 [Kent Yao] Improve CI with cancel & concurrency
dbe8999e [Kent Yao] Improve CI with cancel & concurrency
78a97260 [Kent Yao] Improve CI with cancel & concurrency
7e984ebd [Kent Yao] Improve CI with cancel & concurrency
d2538210 [Kent Yao] Improve CI with cancel & concurrency
51a7c48c [Kent Yao] Improve CI with cancel & concurrency
c5e3177a [Kent Yao] Improve CI with cancel & concurrency
Authored-by: Kent Yao <[email protected]>
Signed-off-by: Kent Yao <[email protected]>
---
.github/workflows/{license.yml => dep.yml} | 40 ++++++++++++++++++++----------
.github/workflows/license.yml | 6 +++++
.github/workflows/master.yml | 7 ++++--
.github/workflows/nightly.yml | 30 +++-------------------
.github/workflows/style.yml | 33 ++++++++++++------------
dev/kyuubi-codecov/pom.xml | 6 +++++
6 files changed, 64 insertions(+), 58 deletions(-)
diff --git a/.github/workflows/license.yml b/.github/workflows/dep.yml
similarity index 57%
copy from .github/workflows/license.yml
copy to .github/workflows/dep.yml
index fa70b31..ee3134a 100644
--- a/.github/workflows/license.yml
+++ b/.github/workflows/dep.yml
@@ -15,31 +15,45 @@
# limitations under the License.
#
-name: License check
-
-# This GitHub workflow checks Apache License v2.0 Header is added if needed
+name: Dependency
on:
pull_request:
branches:
- master
- branch-*
+ paths:
+ # dependency check happens only pom changes
+ - '**/pom.xml'
+
+concurrency:
+ group: dep-${{ github.ref }}
+ cancel-in-progress: true
jobs:
- rat:
- name: License
+ dep:
+ name: Dependency check
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- - name: Setup JDK 8
+ - name: setup java
uses: actions/setup-java@v2
with:
distribution: zulu
java-version: 8
- - run: build/mvn org.apache.rat:apache-rat-plugin:check -Ptpcds
-Pspark-block-cleaner -Pkubernetes-deployment-it -Pspark-3.1 -Pspark-3.2
- - name: Upload rat report
- if: failure()
- uses: actions/upload-artifact@v2
- with:
- name: rat-report
- path: "**/target/rat*.txt"
+ cache: 'maven'
+ check-latest: false
+ - name: build
+ env:
+ MAVEN_OPTS: -Dorg.slf4j.simpleLogger.defaultLogLevel=error
+ run: >-
+ build/mvn clean install
+ -Pflink-provided,spark-provided
+ -Dmaven.javadoc.skip=true
+ -Drat.skip=true
+ -Dscalastyle.skip=true
+ -Dspotless.check.skip
+ -DskipTests
+ -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am
+ - name: Check dependency list
+ run: build/dependency.sh
diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml
index fa70b31..f2cf380 100644
--- a/.github/workflows/license.yml
+++ b/.github/workflows/license.yml
@@ -25,6 +25,10 @@ on:
- master
- branch-*
+concurrency:
+ group: lincense-${{ github.ref }}
+ cancel-in-progress: true
+
jobs:
rat:
name: License
@@ -36,6 +40,8 @@ jobs:
with:
distribution: zulu
java-version: 8
+ cache: 'maven'
+ check-latest: false
- run: build/mvn org.apache.rat:apache-rat-plugin:check -Ptpcds
-Pspark-block-cleaner -Pkubernetes-deployment-it -Pspark-3.1 -Pspark-3.2
- name: Upload rat report
if: failure()
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 38cae0b..af2feee 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -27,6 +27,10 @@ on:
- master
- branch-*
+concurrency:
+ group: test-${{ github.ref }}
+ cancel-in-progress: true
+
jobs:
build:
@@ -95,7 +99,6 @@ jobs:
**/kyuubi-flink-sql-engine.log*
**/kyuubi-spark-sql-engine.log*
**/kyuubi-trino-sql-engine.log*
- **/target/scalastyle-output.xml
tpcds:
name: TPC-DS Tests
@@ -115,7 +118,7 @@ jobs:
check-latest: false
- name: Run TPC-DS Tests
run: >-
- ./build/mvn clean install -Dmaven.javadoc.skip=true -Drat.skip=true
-Dscalastyle.skip=true -Dspotless.check.skip
-Dorg.slf4j.simpleLogger.defaultLogLevel=warn -V
+ ./build/mvn clean install -Dmaven.javadoc.skip=true -Drat.skip=true
-Dscalastyle.skip=true -Dspotless.check.skip
-Dorg.slf4j.simpleLogger.defaultLogLevel=warn
-pl kyuubi-server -am
-Dmaven.plugin.scalatest.exclude.tags=''
-Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index f52805a..bd2c23f 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -15,7 +15,7 @@
# limitations under the License.
#
-name: Kyuubi Nightly
+name: Kyuubi Spark Nightly
on:
schedule:
@@ -41,30 +41,8 @@ jobs:
with:
distribution: zulu
java-version: 8
- - uses: actions/cache@v2
- with:
- path: ~/.m2/repository/com
- key: ${{ runner.os }}-maven-com-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-maven-com-
- - uses: actions/cache@v2
- with:
- path: ~/.m2/repository/org
- key: ${{ runner.os }}-maven-org-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-maven-org-
- - uses: actions/cache@v2
- with:
- path: ~/.m2/repository/net
- key: ${{ runner.os }}-maven-net-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-maven-net-
- - uses: actions/cache@v2
- with:
- path: ~/.m2/repository/io
- key: ${{ runner.os }}-maven-io-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-maven-io-
+ cache: 'maven'
+ check-latest: false
- name: Build with Maven
run: ./build/mvn clean install ${{ matrix.profiles }}
-Dmaven.javadoc.skip=true -V
- name: Upload test logs
@@ -74,6 +52,4 @@ jobs:
name: unit-tests-log
path: |
**/target/unit-tests.log
- **/kyuubi-flink-sql-engine.log*
**/kyuubi-spark-sql-engine.log*
- **/kyuubi-trino-sql-engine.log*
diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml
index 8d995c8..b32ad10 100644
--- a/.github/workflows/style.yml
+++ b/.github/workflows/style.yml
@@ -15,9 +15,7 @@
# limitations under the License.
#
-name: Style check
-
-# This GitHub workflow checks style & dependency issues.
+name: Style
on:
pull_request:
@@ -25,9 +23,18 @@ on:
- master
- branch-*
+concurrency:
+ group: linter-${{ github.ref }}
+ cancel-in-progress: true
+
+# no install runs before style check because it is expensive, we can skip
build if style fails
+# Well, sometimes when we introduce a new module, it may 'fail' the style
check for missing
+# dependency we just create for other module to inherit.
+# We can ignore this failure and merge the PR, if we see the style check pass
on the new module
+# or via a local quick verification.
jobs:
linter:
- name: Style and Dependency check
+ name: Style check
runs-on: ubuntu-20.04
strategy:
matrix:
@@ -40,18 +47,12 @@ jobs:
with:
distribution: zulu
java-version: 8
- - name: Install
- run: >-
- build/mvn clean install -V -Pflink-provided,spark-provided
-Dorg.slf4j.simpleLogger.defaultLogLevel=warn \
- -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true
-Dspotless.check.skip -DskipTests \
- -Pflink-provided,spark-provided \
- -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am
- - name: Scalastyle with Maven
- run: build/mvn scalastyle:check ${{ matrix.profiles }}
+ cache: 'maven'
+ check-latest: false
+ - name: Scalastyle with maven
+ run: build/mvn scalastyle:check ${{ matrix.profiles }} -pl
'!:kyuubi-codecov_2.12,!:kyuubi-assembly_2.12'
- name: Upload scalastyle report
if: failure()
run: for log in `find * -name "scalastyle-output.xml"`; do echo
"=========$log========="; grep "error" $log; done
- - name: JavaStyle with Maven
- run: build/mvn spotless:check ${{ matrix.profiles }}
- - name: Check dependency list
- run: build/dependency.sh
+ - name: JavaStyle with maven
+ run: build/mvn spotless:check ${{ matrix.profiles }} -pl
'!:kyuubi-codecov_2.12,!:kyuubi-assembly_2.12'
diff --git a/dev/kyuubi-codecov/pom.xml b/dev/kyuubi-codecov/pom.xml
index f844944..3672ab4 100644
--- a/dev/kyuubi-codecov/pom.xml
+++ b/dev/kyuubi-codecov/pom.xml
@@ -110,6 +110,12 @@
<artifactId>kyuubi-hive-beeline</artifactId>
<version>${project.version}</version>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.kyuubi</groupId>
+
<artifactId>kyuubi-hive-sql-engine_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
<build>
![https://user-images.githubusercontent.com/8326978/159059722-480f4fd6-d87e-4b07-984e-eb5b08a84beb.png"](https://user-images.githubusercontent.com/8326978/159059722-480f4fd6-d87e-4b07-984e-eb5b08a84beb.png"){kind=link}