This is an automated email from the ASF dual-hosted git repository.
pingsutw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 05d6430 SUBMARINE-980. Upload code coverage to SonarCloud
05d6430 is described below
commit 05d6430c625780d71b5abeeb51a2acafe6ac7e2d
Author: Lisa <[email protected]>
AuthorDate: Sat Sep 11 20:03:03 2021 +0800
SUBMARINE-980. Upload code coverage to SonarCloud
### What is this PR for?
Upload code coverage to SonarCloud
### What type of PR is it?
[Improvement]
### Todos
* [ ] - Task
### What is the Jira issue?
https://issues.apache.org/jira/projects/SUBMARINE/issues/SUBMARINE-980
### How should this be tested?
### Screenshots (if appropriate)
### Questions:
* Do the license files need updating? No
* Are there breaking changes for older versions? No
* Does this need new documentation? No
Author: Lisa <[email protected]>
Signed-off-by: Kevin <[email protected]>
Closes #742 from aeioulisa/SUBMARINE-980 and squashes the following commits:
fc126de7 [Lisa] running sonar job only on push to master
3bcb2f6b [Lisa] Upload code coverage to SonarCloud
f52493b1 [Lisa] Merge branch 'master' of https://github.com/apache/submarine
020117ea [Lisa] update experiment-info.component.html
---
.github/workflows/master.yml | 173 ++++++++++++++++++++++++++++++++++++++-
.github/workflows/sonarcloud.yml | 62 --------------
dev-support/cicd/coverage.sh | 52 ++++++++++++
pom.xml | 23 ++++++
4 files changed, 247 insertions(+), 63 deletions(-)
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 0cc7147..1777f3e 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -42,6 +42,11 @@ jobs:
- uses: actions/checkout@v2
with:
fetch-depth: 50
+ - name: Cache jacoco.exec
+ uses: actions/cache@v2
+ with:
+ path: ./submarine-test/test-e2e/target/jacoco.exec
+ key: ${{ runner.os }}-docker-${{ github.sha }}
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:
@@ -78,6 +83,11 @@ jobs:
- uses: actions/checkout@v2
with:
fetch-depth: 50
+ - name: Cache jacoco.exec
+ uses: actions/cache@v2
+ with:
+ path: ./submarine-test/test-k8s/target/jacoco.exec
+ key: ${{ runner.os }}-docker-${{ github.sha }}
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:
@@ -140,6 +150,11 @@ jobs:
- uses: actions/checkout@v2
with:
fetch-depth: 50
+ - name: Cache jacoco.exec
+ uses: actions/cache@v2
+ with:
+ path: ./submarine-commons/commons-cluster/target/jacoco.exec
+ key: ${{ runner.os }}-docker-${{ github.sha }}
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:
@@ -180,6 +195,11 @@ jobs:
- uses: actions/checkout@v2
with:
fetch-depth: 50
+ - name: Cache jacoco.exec
+ uses: actions/cache@v2
+ with:
+ path: ./submarine-commons/commons-metastore/target/jacoco.exec
+ key: ${{ runner.os }}-docker-${{ github.sha }}
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:
@@ -216,6 +236,11 @@ jobs:
- uses: actions/checkout@v2
with:
fetch-depth: 50
+ - name: Cache jacoco.exec
+ uses: actions/cache@v2
+ with:
+ path: ./submarine-commons/commons-rpc/target/jacoco.exec
+ key: ${{ runner.os }}-docker-${{ github.sha }}
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:
@@ -247,6 +272,11 @@ jobs:
- uses: actions/checkout@v2
with:
fetch-depth: 50
+ - name: Cache jacoco.exec
+ uses: actions/cache@v2
+ with:
+ path: ./submarine-commons/commons-runtime/target/jacoco.exec
+ key: ${{ runner.os }}-docker-${{ github.sha }}
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:
@@ -278,6 +308,11 @@ jobs:
- uses: actions/checkout@v2
with:
fetch-depth: 50
+ - name: Cache jacoco.exec
+ uses: actions/cache@v2
+ with:
+ path: ./submarine-commons/commons-unixusersync/target/jacoco.exec
+ key: ${{ runner.os }}-docker-${{ github.sha }}
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:
@@ -309,6 +344,11 @@ jobs:
- uses: actions/checkout@v2
with:
fetch-depth: 50
+ - name: Cache jacoco.exec
+ uses: actions/cache@v2
+ with:
+ path: ./submarine-client/target/jacoco.exec
+ key: ${{ runner.os }}-docker-${{ github.sha }}
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:
@@ -349,6 +389,13 @@ jobs:
- uses: actions/checkout@v2
with:
fetch-depth: 50
+ - name: Cache jacoco.exec
+ uses: actions/cache@v2
+ with:
+ path: |
+ ./submarine-server/server-rpc/target/jacoco.exec
+ ./submarine-server/server-core/target/jacoco.exec
+ key: ${{ runner.os }}-docker-${{ github.sha }}
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:
@@ -415,6 +462,13 @@ jobs:
- uses: actions/checkout@v2
with:
fetch-depth: 50
+ - name: Cache jacoco.exec
+ uses: actions/cache@v2
+ with:
+ path: |
+ ./submarine-server/server-submitter/target/jacoco.exec
+ ./submarine-server/server-submitter/submitter-yarn/target/jacoco.exec
+ key: ${{ runner.os }}-docker-${{ github.sha }}
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:
@@ -484,7 +538,6 @@ jobs:
with:
java-version: '1.8'
- run: mvn org.apache.rat:apache-rat-plugin:check
-
linter:
name: Check Style
runs-on: ubuntu-latest
@@ -515,3 +568,121 @@ jobs:
run: ./dev-support/style-check/lint-scala.sh
- name: Angular Style
run: ./dev-support/style-check/lint-angular.sh
+ sonarcloud:
+ if: github.repository == 'apache/submarine' && github.event_name == 'push'
&& github.ref == 'refs/heads/master'
+ name: SonarCloud
+ runs-on: ubuntu-latest
+ needs:
+ - submarine-e2e
+ - submarine-k8s
+ - submarine-security
+ - submarine-submitter
+ - submarine-server
+ - submarine-client
+ - submarine-commons-unixusersync
+ - submarine-commons-runtime
+ - submarine-commons-rpc
+ - submarine-commons-metastore
+ - submarine-commons-cluster
+ steps:
+ - uses: actions/checkout@v2
+ with:
+ fetch-depth: 0 # Shallow clones should be disabled for a better
relevancy of analysis
+ - name: Cache Maven packages
+ uses: actions/cache@v1
+ with:
+ path: ~/.m2
+ key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+ restore-keys: ${{ runner.os }}-m2
+ - name: Cache test-e2e data
+ uses: actions/cache@v2
+ with:
+ path: ./submarine-test/test-e2e/target/jacoco.exec
+ key: ${{ runner.os }}-docker-${{ github.sha }}
+ - name: Cache test-k8s data
+ uses: actions/cache@v2
+ with:
+ path: ./submarine-test/test-k8s/target/jacoco.exec
+ key: ${{ runner.os }}-docker-${{ github.sha }}
+ - name: Cache commons-cluster data
+ uses: actions/cache@v2
+ with:
+ path: ./submarine-commons/commons-cluster/target/jacoco.exec
+ key: ${{ runner.os }}-docker-${{ github.sha }}
+ - name: Cache commons-metastore data
+ uses: actions/cache@v2
+ with:
+ path: ./submarine-commons/commons-metastore/target/jacoco.exec
+ key: ${{ runner.os }}-docker-${{ github.sha }}
+ - name: Cache commons-rpc data
+ uses: actions/cache@v2
+ with:
+ path: ./submarine-commons/commons-rpc/target/jacoco.exec
+ key: ${{ runner.os }}-docker-${{ github.sha }}
+ - name: Cache commons-runtime data
+ uses: actions/cache@v2
+ with:
+ path: ./submarine-commons/commons-runtime/target/jacoco.exec
+ key: ${{ runner.os }}-docker-${{ github.sha }}
+ - name: Cache commons-unixusersync data
+ uses: actions/cache@v2
+ with:
+ path: ./submarine-commons/commons-unixusersync/target/jacoco.exec
+ key: ${{ runner.os }}-docker-${{ github.sha }}
+ - name: Cache submarine-client data
+ uses: actions/cache@v2
+ with:
+ path: ./submarine-client/target/jacoco.exec
+ key: ${{ runner.os }}-docker-${{ github.sha }}
+ - name: Cache submarine-server data
+ uses: actions/cache@v2
+ with:
+ path: |
+ ./submarine-server/server-rpc/target/jacoco.exec
+ ./submarine-server/server-core/target/jacoco.exec
+ key: ${{ runner.os }}-docker-${{ github.sha }}
+ - name: Cache submarine-submitter data
+ uses: actions/cache@v2
+ with:
+ path: |
+ ./submarine-server/server-submitter/target/jacoco.exec
+
./submarine-server/server-submitter/submitter-yarn/target/jacoco.exec
+ key: ${{ runner.os }}-docker-${{ github.sha }}
+ - name: Cache SonarCloud packages
+ uses: actions/cache@v1
+ with:
+ path: ~/.sonar/cache
+ key: ${{ runner.os }}-sonar
+ restore-keys: ${{ runner.os }}-sonar
+ - name: Set up JDK 1.8
+ uses: actions/setup-java@v1
+ with:
+ java-version: "1.8"
+ - name: Set up Maven 3.6.3
+ uses: stCarolas/setup-maven@v4
+ with:
+ maven-version: 3.6.3
+ - name: Build the project with JDK 8
+ run: mvn install -DskipTests
+ - name: Set up JDK 11
+ uses: actions/setup-java@v1
+ with:
+ java-version: 11
+ - name: Download artifacts
+ uses: actions/download-artifact@v2
+ with:
+ path: target/artifacts
+ - name: Calculate combined coverage
+ run: ./dev-support/cicd/coverage.sh
+ - name: Build and analyze with JDK 11 # sonar-maven-plugin only support
JDK 11
+ run:
+ mvn -B verify -DskipTests
org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -pl $EXCLUDE_MODULE
-Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=apache
-Dsonar.projectKey=apache_submarine
+ env:
+ SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ EXCLUDE_MODULE: '!:submarine-commons-rpc,!:submarine-spark-security'
# Exclude the modules that can't be compiled with JDK 11
+ - name: Delete temporary build artifacts before caching
+ run: |
+ #Never cache local artifacts
+ rm -rf ~/.m2/repository/org/apache/submarine
+ if: always()
diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml
deleted file mode 100644
index 04b1c9e..0000000
--- a/.github/workflows/sonarcloud.yml
+++ /dev/null
@@ -1,62 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: SonarCloud
-on:
- push:
- branches:
- - master
-
-jobs:
- sonarcloud:
- if: github.repository == 'apache/submarine'
- name: SonarCloud
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v2
- with:
- fetch-depth: 0 # Shallow clones should be disabled for a better
relevancy of analysis
- - name: Cache Maven packages
- uses: actions/cache@v1
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2
- - name: Cache SonarCloud packages
- uses: actions/cache@v1
- with:
- path: ~/.sonar/cache
- key: ${{ runner.os }}-sonar
- restore-keys: ${{ runner.os }}-sonar
- - name: Set up JDK 1.8
- uses: actions/setup-java@v1
- with:
- java-version: "1.8"
- - name: Set up Maven 3.6.3
- uses: stCarolas/setup-maven@v4
- with:
- maven-version: 3.6.3
- - name: Build the project with JDK 8
- run: mvn clean install -DskipTests
- - name: Set up JDK 11
- uses: actions/setup-java@v1
- with:
- java-version: 11
- - name: Build and analyze with JDK 11 # sonar-maven-plugin only support
JDK 11
- run: mvn verify -DskipTests
org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -pl $EXCLUDE_MODULE
-Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=apache
-Dsonar.projectKey=apache_submarine
- env:
- SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- EXCLUDE_MODULE: '!:submarine-commons-rpc,!:submarine-spark-security'
# Exclude the modules that can't be compiled with JDK 11
diff --git a/dev-support/cicd/coverage.sh b/dev-support/cicd/coverage.sh
new file mode 100755
index 0000000..44ea642
--- /dev/null
+++ b/dev-support/cicd/coverage.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script merges the combined jacoco files (output of unit.sh and others)
+# and generates a report in HTML and XML formats
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+cd "$DIR/../.." || exit 1
+
+set -ex
+
+REPORT_DIR="$DIR/../../target/coverage"
+
+mkdir -p "$REPORT_DIR"
+
+#Install jacoco cli
+mvn --non-recursive --no-transfer-progress \
+ org.apache.maven.plugins:maven-dependency-plugin:3.1.2:copy \
+ -Dartifact=org.jacoco:org.jacoco.cli:0.8.0:jar:nodeps
+
+jacoco() {
+ java -jar target/dependency/org.jacoco.cli-0.8.0-nodeps.jar "$@"
+}
+
+#Merge all the jacoco.exec files
+jacoco merge $(find $DIR/../.. -name 'jacoco*.exec') --destfile
"$REPORT_DIR/jacoco-all.exec"
+
+rm -rf target/coverage-classes || true
+mkdir -p target/coverage-classes
+
+#Unzip all the classes from the last build
+find submarine-dist/target/* -name 'submarine-all*.jar' | \
+ xargs -n1 unzip -o -q -d target/coverage-classes
+
+#Exclude some classes from the coverage
+find target/coverage-classes/javax/xml/bind -name 'ModuleUtil.class'| xargs rm
-rf
+
+#generate the reports
+jacoco report "$REPORT_DIR/jacoco-all.exec" --classfiles
target/coverage-classes --html "$REPORT_DIR/all" --xml "$REPORT_DIR/all.xml"
diff --git a/pom.xml b/pom.xml
index d173543..3bad9a7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -571,6 +571,29 @@
</excludes>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.jacoco</groupId>
+ <artifactId>jacoco-maven-plugin</artifactId>
+ <version>0.8.0</version>
+ <configuration>
+ <destFile>${basedir}/target/jacoco.exec</destFile>
+ <dataFile>${basedir}/target/jacoco.exec</dataFile>
+ </configuration>
+ <executions>
+ <execution>
+ <id>default-prepare-agent</id>
+ <goals>
+ <goal>prepare-agent</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>report</id>
+ <goals>
+ <goal>report</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
