This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 1e04c8a095 separate pr test scripts and codecov (#11804)
1e04c8a095 is described below
commit 1e04c8a095d916c0a473ece7186c668b996eb302
Author: Xiang Fu <[email protected]>
AuthorDate: Fri Oct 13 17:07:19 2023 -0700
separate pr test scripts and codecov (#11804)
---
.github/workflows/pinot_tests.yml | 59 ++++++++++++++-
.github/workflows/scripts/.pinot_test.sh | 84 ----------------------
.../scripts/pr-tests/.pinot_tests_build.sh | 62 ++++++++++++++++
.../pr-tests/.pinot_tests_custom_integration.sh | 33 +++++++++
.../scripts/pr-tests/.pinot_tests_integration.sh | 37 ++++++++++
.../scripts/pr-tests/.pinot_tests_unit.sh | 52 ++++++++++++++
6 files changed, 241 insertions(+), 86 deletions(-)
diff --git a/.github/workflows/pinot_tests.yml
b/.github/workflows/pinot_tests.yml
index 07750f239b..727280bd2e 100644
--- a/.github/workflows/pinot_tests.yml
+++ b/.github/workflows/pinot_tests.yml
@@ -97,6 +97,21 @@ jobs:
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
+ - name: Build Project
+ env:
+ RUN_INTEGRATION_TESTS: false
+ RUN_TEST_SET: ${{ matrix.testset }}
+ GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ MAVEN_OPTS: >
+ -Xmx2G -DskipShade -DfailIfNoTests=false
-Dmaven.wagon.httpconnectionManager.ttlSeconds=25
+ -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false
-Dmaven.wagon.http.pool=false
+ -XX:+IgnoreUnrecognizedVMOptions
+ --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
+ --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+ --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
+ --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
+ --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+ run: .github/workflows/scripts/pr-tests/.pinot_tests_build.sh
- name: Unit Test
env:
RUN_INTEGRATION_TESTS: false
@@ -111,7 +126,7 @@ jobs:
--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
- run: .github/workflows/scripts/.pinot_test.sh
+ run: .github/workflows/scripts/pr-tests/.pinot_tests_unit.sh
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
continue-on-error: true
@@ -150,6 +165,20 @@ jobs:
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
+ - name: Build Project
+ env:
+ RUN_INTEGRATION_TESTS: true
+ GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ MAVEN_OPTS: >
+ -Xmx2G -DskipShade -DfailIfNoTests=false
-Dmaven.wagon.httpconnectionManager.ttlSeconds=25
+ -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false
-Dmaven.wagon.http.pool=false
+ -XX:+IgnoreUnrecognizedVMOptions
+ --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
+ --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+ --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
+ --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
+ --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+ run: .github/workflows/scripts/pr-tests/.pinot_tests_build.sh
- name: Integration Test
env:
RUN_INTEGRATION_TESTS: true
@@ -164,7 +193,7 @@ jobs:
--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
- run: .github/workflows/scripts/.pinot_test.sh
+ run: .github/workflows/scripts/pr-tests/.pinot_tests_integration.sh
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
continue-on-error: true
@@ -174,6 +203,32 @@ jobs:
name: codecov-integration-tests
fail_ci_if_error: false
verbose: true
+ - name: Custom Integration Test
+ if : ${{ matrix.testset == 1 }}
+ env:
+ RUN_INTEGRATION_TESTS: true
+ RUN_TEST_SET: ${{ matrix.testset }}
+ GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ MAVEN_OPTS: >
+ -Xmx2G -DskipShade -DfailIfNoTests=false
-Dmaven.wagon.httpconnectionManager.ttlSeconds=25
+ -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false
-Dmaven.wagon.http.pool=false
+ -XX:+IgnoreUnrecognizedVMOptions
+ --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
+ --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+ --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
+ --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
+ --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+ run:
.github/workflows/scripts/pr-tests/.pinot_tests_custom_integration.sh
+ - name: Upload coverage to Codecov
+ if : ${{ matrix.testset == 1 }}
+ uses: codecov/codecov-action@v3
+ continue-on-error: true
+ timeout-minutes: 5
+ with:
+ flags: integration,custom-integration${{ matrix.testset
}},${{matrix.distribution}},java-${{matrix.java}}
+ name: codecov-custom-integration-tests
+ fail_ci_if_error: false
+ verbose: true
compatibility-verifier:
if: github.repository == 'apache/pinot'
diff --git a/.github/workflows/scripts/.pinot_test.sh
b/.github/workflows/scripts/.pinot_test.sh
deleted file mode 100755
index 7179fa91a0..0000000000
--- a/.github/workflows/scripts/.pinot_test.sh
+++ /dev/null
@@ -1,84 +0,0 @@
-#!/bin/bash -x
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# Java version
-java -version
-
-# Check network
-ifconfig
-netstat -i
-
-if [ "$RUN_INTEGRATION_TESTS" != false ]; then
- # Integration Tests
- mvn clean install -DskipTests -Dcheckstyle.skip -Dspotless.skip
-Denforcer.skip -Dlicense.skip -am -B \
- -pl 'pinot-integration-tests' -T 16 || exit 1
- cd pinot-integration-tests || exit 1
- if [ "$RUN_TEST_SET" == "1" ]; then
- mvn test \
- -P github-actions,custom-cluster-integration-test-suite || exit 1
- mvn test \
- -P github-actions,integration-tests-set-1 && exit 0 || exit 1
- fi
- if [ "$RUN_TEST_SET" == "2" ]; then
- mvn test \
- -P github-actions,integration-tests-set-2 && exit 0 || exit 1
- fi
-else
- # Unit Tests
- # - TEST_SET#1 runs install and test together so the module list must
ensure no additional modules were tested
- # due to the -am flag (include dependency modules)
- if [ "$RUN_TEST_SET" == "1" ]; then
- mvn clean install -DskipTests -am -B -T 16 \
- -pl 'pinot-spi' \
- -pl 'pinot-segment-spi' \
- -pl 'pinot-common' \
- -pl 'pinot-segment-local' \
- -pl 'pinot-core' \
- -pl 'pinot-query-planner' \
- -pl 'pinot-query-runtime' \
- -P github-actions,no-integration-tests \
- -Dcheckstyle.skip -Dspotless.skip -Denforcer.skip -Dlicense.skip ||
exit 1
- mvn test -T 16 \
- -pl 'pinot-spi' \
- -pl 'pinot-segment-spi' \
- -pl 'pinot-common' \
- -pl 'pinot-segment-local' \
- -pl 'pinot-core' \
- -pl 'pinot-query-planner' \
- -pl 'pinot-query-runtime' \
- -P github-actions,no-integration-tests && exit 0 || exit 1
- fi
- if [ "$RUN_TEST_SET" == "2" ]; then
- mvn clean install -DskipTests -Dcheckstyle.skip -Dspotless.skip
-Denforcer.skip -Dlicense.skip -T 16 || exit 1
- mvn test -am -B \
- -pl '!pinot-spi' \
- -pl '!pinot-segment-spi' \
- -pl '!pinot-common' \
- -pl '!pinot-segment-local' \
- -pl '!pinot-core' \
- -pl '!pinot-query-planner' \
- -pl '!pinot-query-runtime' \
- -P github-actions,no-integration-tests \
- -Dspotless.skip -Dcheckstyle.skip -Denforcer.skip -Dlicense.skip \
- && exit 0 || exit 1
- fi
-fi
-
-mvn clean > /dev/null
diff --git a/.github/workflows/scripts/pr-tests/.pinot_tests_build.sh
b/.github/workflows/scripts/pr-tests/.pinot_tests_build.sh
new file mode 100755
index 0000000000..05080fd963
--- /dev/null
+++ b/.github/workflows/scripts/pr-tests/.pinot_tests_build.sh
@@ -0,0 +1,62 @@
+#!/bin/bash -x
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Java version
+java -version
+
+# Check network
+ifconfig
+netstat -i
+
+if [ "$RUN_INTEGRATION_TESTS" != false ]; then
+ # Integration Tests
+ mvn clean install \
+ -DskipTests -Dcheckstyle.skip -Dspotless.skip -Denforcer.skip
-Dlicense.skip -Dmaven.plugin.appassembler.skip=true \
+ -am -B -T 16 \
+ -P github-actions,integration-tests \
+ -pl 'pinot-integration-tests' || exit 1
+else
+ # Unit Tests
+ # - TEST_SET#1 runs install and test together so the module list must
ensure no additional modules were tested
+ # due to the -am flag (include dependency modules)
+ if [ "$RUN_TEST_SET" == "1" ]; then
+ mvn clean install \
+ -DskipTests -Dcheckstyle.skip -Dspotless.skip -Denforcer.skip
-Dlicense.skip -Dmaven.plugin.appassembler.skip=true \
+ -am -B -T 16 \
+ -P github-actions,no-integration-tests \
+ -pl 'pinot-spi' \
+ -pl 'pinot-segment-spi' \
+ -pl 'pinot-common' \
+ -pl 'pinot-segment-local' \
+ -pl 'pinot-core' \
+ -pl 'pinot-query-planner' \
+ -pl 'pinot-query-runtime' || exit 1
+ fi
+ if [ "$RUN_TEST_SET" == "2" ]; then
+ mvn clean install \
+ -DskipTests -Dcheckstyle.skip -Dspotless.skip -Denforcer.skip
-Dlicense.skip -Dmaven.plugin.appassembler.skip=true \
+ -am -B -T 16 \
+ -P github-actions,no-integration-tests \
+ -pl '!pinot-integration-test-base' \
+ -pl '!pinot-integration-tests' \
+ -pl '!pinot-perf' \
+ -pl '!pinot-distribution' || exit 1
+ fi
+fi
diff --git
a/.github/workflows/scripts/pr-tests/.pinot_tests_custom_integration.sh
b/.github/workflows/scripts/pr-tests/.pinot_tests_custom_integration.sh
new file mode 100755
index 0000000000..cd99615ce4
--- /dev/null
+++ b/.github/workflows/scripts/pr-tests/.pinot_tests_custom_integration.sh
@@ -0,0 +1,33 @@
+#!/bin/bash -x
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Java version
+java -version
+
+# Check network
+ifconfig
+netstat -i
+
+# Custom Integration Tests
+cd pinot-integration-tests || exit 1
+if [ "$RUN_TEST_SET" == "1" ]; then
+ mvn test \
+ -P github-actions,custom-cluster-integration-test-suite || exit 1
+fi
diff --git a/.github/workflows/scripts/pr-tests/.pinot_tests_integration.sh
b/.github/workflows/scripts/pr-tests/.pinot_tests_integration.sh
new file mode 100755
index 0000000000..a6f671c218
--- /dev/null
+++ b/.github/workflows/scripts/pr-tests/.pinot_tests_integration.sh
@@ -0,0 +1,37 @@
+#!/bin/bash -x
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Java version
+java -version
+
+# Check network
+ifconfig
+netstat -i
+
+# Integration Tests
+cd pinot-integration-tests || exit 1
+if [ "$RUN_TEST_SET" == "1" ]; then
+ mvn test \
+ -P github-actions,integration-tests-set-1 && exit 0 || exit 1
+fi
+if [ "$RUN_TEST_SET" == "2" ]; then
+ mvn test \
+ -P github-actions,integration-tests-set-2 && exit 0 || exit 1
+fi
diff --git a/.github/workflows/scripts/pr-tests/.pinot_tests_unit.sh
b/.github/workflows/scripts/pr-tests/.pinot_tests_unit.sh
new file mode 100755
index 0000000000..3a68ff722c
--- /dev/null
+++ b/.github/workflows/scripts/pr-tests/.pinot_tests_unit.sh
@@ -0,0 +1,52 @@
+#!/bin/bash -x
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Java version
+java -version
+
+# Check network
+ifconfig
+netstat -i
+
+# Unit Tests
+# - TEST_SET#1 runs install and test together so the module list must ensure
no additional modules were tested
+# due to the -am flag (include dependency modules)
+if [ "$RUN_TEST_SET" == "1" ]; then
+ mvn test -T 16 \
+ -pl 'pinot-spi' \
+ -pl 'pinot-segment-spi' \
+ -pl 'pinot-common' \
+ -pl 'pinot-segment-local' \
+ -pl 'pinot-core' \
+ -pl 'pinot-query-planner' \
+ -pl 'pinot-query-runtime' \
+ -P github-actions,no-integration-tests || exit 1
+fi
+if [ "$RUN_TEST_SET" == "2" ]; then
+ mvn test \
+ -pl '!pinot-spi' \
+ -pl '!pinot-segment-spi' \
+ -pl '!pinot-common' \
+ -pl '!pinot-segment-local' \
+ -pl '!pinot-core' \
+ -pl '!pinot-query-planner' \
+ -pl '!pinot-query-runtime' \
+ -P github-actions,no-integration-tests || exit 1
+fi
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
