This is an automated email from the ASF dual-hosted git repository. lhotari pushed a commit to branch branch-3.1 in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/branch-3.1 by this push: new 1b4b92082fa [fix][ci] Fix OWASP Dependency Check download by using NVD API key (#22999) 1b4b92082fa is described below commit 1b4b92082fa2bfd0cabd6324489c42bdcba1d207 Author: Lari Hotari <lhot...@users.noreply.github.com> AuthorDate: Thu Jul 4 12:41:21 2024 +0300 [fix][ci] Fix OWASP Dependency Check download by using NVD API key (#22999) (cherry picked from commit 8b7754f11f113af9d341a460795d0c7b8095f594) --- .github/workflows/ci-owasp-dependency-check.yaml | 81 +++++++++++++++++------- .github/workflows/pulsar-ci.yaml | 5 +- distribution/io/pom.xml | 1 - pom.xml | 14 +++- pulsar-io/docs/pom.xml | 1 - pulsar-io/flume/pom.xml | 1 - pulsar-io/hbase/pom.xml | 1 - pulsar-io/hdfs2/pom.xml | 7 +- pulsar-io/hdfs3/pom.xml | 9 ++- tiered-storage/file-system/pom.xml | 1 - 10 files changed, 78 insertions(+), 43 deletions(-) diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 69eb0a7a198..a70f4a82ff1 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -24,7 +24,9 @@ on: workflow_dispatch: env: - MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + JDK_DISTRIBUTION: corretto + NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }} jobs: run-owasp-dependency-check: @@ -34,65 +36,96 @@ jobs: JOB_NAME: Check ${{ matrix.branch }} GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} runs-on: ubuntu-22.04 - timeout-minutes: 45 + timeout-minutes: 75 strategy: fail-fast: false + max-parallel: 1 matrix: include: - branch: master + - branch: branch-3.3 + - branch: branch-3.2 - branch: branch-3.0 - - branch: branch-2.11 - - branch: branch-2.10 - jdk: 11 - - branch: branch-2.9 - jdk: 11 - - branch: branch-2.8 - jdk: 11 steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ matrix.branch }} - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Cache local Maven repository - uses: actions/cache@v3 + - name: Restore Maven repository cache + uses: actions/cache/restore@v4 timeout-minutes: 5 with: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-owasp-${{ hashFiles('**/pom.xml') }} + key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} restore-keys: | - ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK ${{ matrix.jdk || '17' }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: - distribution: 'temurin' + distribution: ${{ env.JDK_DISTRIBUTION }} java-version: ${{ matrix.jdk || '17' }} - name: run install by skip tests - run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true + run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true -DnarPluginPhase=none -pl '!distribution/io,!distribution/offloaders' + + - name: OWASP cache key weeknum + id: get-weeknum + run: | + echo "weeknum=$(date -u +"%Y-%U")" >> $GITHUB_OUTPUT + shell: bash + + - name: Restore OWASP Dependency Check data + id: restore-owasp-dependency-check-data + uses: actions/cache/restore@v4 + timeout-minutes: 5 + with: + path: ~/.m2/repository/org/owasp/dependency-check-data + key: owasp-dependency-check-data-${{ steps.get-weeknum.outputs.weeknum }} + enableCrossOsArchive: true + restore-keys: | + owasp-dependency-check-data- + + - name: Update OWASP Dependency Check data + id: update-owasp-dependency-check-data + if: ${{ matrix.branch == 'master' && (steps.restore-owasp-dependency-check-data.outputs.cache-hit != 'true' || steps.restore-owasp-dependency-check-data.outputs.cache-matched-key != steps.restore-owasp-dependency-check-data.outputs.cache-primary-key) }} + run: mvn -B -ntp -Powasp-dependency-check initialize -pl . dependency-check:update-only + + - name: Save OWASP Dependency Check data + if: ${{ steps.update-owasp-dependency-check-data.outcome == 'success' }} + uses: actions/cache/save@v4 + timeout-minutes: 5 + with: + path: ~/.m2/repository/org/owasp/dependency-check-data + key: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-primary-key }} + enableCrossOsArchive: true - name: run OWASP Dependency Check for distribution/server (-DfailBuildOnAnyVulnerability=true) run: mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl distribution/server -DfailBuildOnAnyVulnerability=true - - name: run OWASP Dependency Check for distribution/offloaders, distribution/io and pulsar-sql/presto-distribution - run: mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl distribution/offloaders,distribution/io,pulsar-sql/presto-distribution + - name: run OWASP Dependency Check for offloaders/tiered-storage and pulsar-io connectors (-DfailOnError=false) + if: ${{ !cancelled() }} + run: | + mvnprojects=$(mvn -B -ntp -Dscan=false initialize \ + | grep -- "-< .* >-" \ + | sed -E 's/.*-< (.*) >-.*/\1/' \ + | grep -E 'pulsar-io-|tiered-storage-|offloader' \ + | tr '\n' ',' | sed 's/,$/\n/' ) + set -xe + mvn --fail-at-end -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -DfailOnError=false -pl "${mvnprojects}" - name: Upload OWASP Dependency Check reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: owasp-dependency-check-reports-${{ matrix.branch }} path: | - distribution/server/target/dependency-check-report.html - distribution/offloaders/target/dependency-check-report.html - distribution/io/target/dependency-check-report.html - pulsar-sql/presto-distribution/target/dependency-check-report.html + **/target/dependency-check-report.html \ No newline at end of file diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 8aaacd5a8e7..89736a0b13b 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -1250,6 +1250,7 @@ jobs: if: ${{ needs.preconditions.outputs.need_owasp == 'true' }} env: GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }} steps: - name: checkout uses: actions/checkout@v3 @@ -1265,8 +1266,8 @@ jobs: with: limit-access-to-actor: true - - name: Cache Maven dependencies - uses: actions/cache@v3 + - name: Restore Maven repository cache + uses: actions/cache/restore@v4 timeout-minutes: 5 with: path: | diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index c56547c0a65..17f8453194a 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -137,7 +137,6 @@ <plugin> <groupId>org.owasp</groupId> <artifactId>dependency-check-maven</artifactId> - <version>${dependency-check-maven.version}</version> <executions> <execution> <goals> diff --git a/pom.xml b/pom.xml index 5892b2b090c..5cff1a421af 100644 --- a/pom.xml +++ b/pom.xml @@ -304,7 +304,7 @@ flexible messaging model and an intuitive client API.</description> <errorprone-slf4j.version>0.1.4</errorprone-slf4j.version> <j2objc-annotations.version>1.3</j2objc-annotations.version> <lightproto-maven-plugin.version>0.4</lightproto-maven-plugin.version> - <dependency-check-maven.version>9.1.0</dependency-check-maven.version> + <dependency-check-maven.version>10.0.1</dependency-check-maven.version> <roaringbitmap.version>0.9.44</roaringbitmap.version> <extra-enforcer-rules.version>1.6.1</extra-enforcer-rules.version> <oshi.version>6.4.0</oshi.version> @@ -2122,6 +2122,16 @@ flexible messaging model and an intuitive client API.</description> <artifactId>build-helper-maven-plugin</artifactId> <version>${build-helper-maven-plugin.version}</version> </plugin> + <plugin> + <groupId>org.owasp</groupId> + <artifactId>dependency-check-maven</artifactId> + <version>${dependency-check-maven.version}</version> + <configuration> + <nvdApiKeyEnvironmentVariable>NIST_NVD_API_KEY</nvdApiKeyEnvironmentVariable> + <!-- Uncomment the following to use the NVD data feed provided by the Dependency-Check project --> + <!-- <nvdDatafeedUrl>https://jeremylong.github.io/DependencyCheck/hb_nvd/</nvdDatafeedUrl> --> + </configuration> + </plugin> </plugins> </pluginManagement> <extensions> @@ -2553,7 +2563,6 @@ flexible messaging model and an intuitive client API.</description> <plugin> <groupId>org.owasp</groupId> <artifactId>dependency-check-maven</artifactId> - <version>${dependency-check-maven.version}</version> <configuration> <suppressionFiles> <suppressionFile>${pulsar.basedir}/src/owasp-dependency-check-false-positives.xml</suppressionFile> @@ -2588,7 +2597,6 @@ flexible messaging model and an intuitive client API.</description> <plugin> <groupId>org.owasp</groupId> <artifactId>dependency-check-maven</artifactId> - <version>${dependency-check-maven.version}</version> <reportSets> <reportSet> <reports> diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml index d162d8824b9..2642ecd4609 100644 --- a/pulsar-io/docs/pom.xml +++ b/pulsar-io/docs/pom.xml @@ -253,7 +253,6 @@ <plugin> <groupId>org.owasp</groupId> <artifactId>dependency-check-maven</artifactId> - <version>${dependency-check-maven.version}</version> <executions> <execution> <goals> diff --git a/pulsar-io/flume/pom.xml b/pulsar-io/flume/pom.xml index 5b68a802a7d..50ac33ce466 100644 --- a/pulsar-io/flume/pom.xml +++ b/pulsar-io/flume/pom.xml @@ -145,7 +145,6 @@ <plugin> <groupId>org.owasp</groupId> <artifactId>dependency-check-maven</artifactId> - <version>${dependency-check-maven.version}</version> <executions> <execution> <goals> diff --git a/pulsar-io/hbase/pom.xml b/pulsar-io/hbase/pom.xml index 3b91b0f1354..aa46b4a773f 100644 --- a/pulsar-io/hbase/pom.xml +++ b/pulsar-io/hbase/pom.xml @@ -108,7 +108,6 @@ <plugin> <groupId>org.owasp</groupId> <artifactId>dependency-check-maven</artifactId> - <version>${dependency-check-maven.version}</version> <executions> <execution> <goals> diff --git a/pulsar-io/hdfs2/pom.xml b/pulsar-io/hdfs2/pom.xml index 33630126c49..5e12160ed59 100644 --- a/pulsar-io/hdfs2/pom.xml +++ b/pulsar-io/hdfs2/pom.xml @@ -27,14 +27,14 @@ </parent> <artifactId>pulsar-io-hdfs2</artifactId> <name>Pulsar IO :: Hdfs2</name> - + <dependencies> <dependency> <groupId>${project.groupId}</groupId> <artifactId>pulsar-io-core</artifactId> <version>${project.version}</version> </dependency> - + <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> @@ -74,7 +74,7 @@ <artifactId>commons-lang3</artifactId> </dependency> </dependencies> - + <build> <plugins> <plugin> @@ -113,7 +113,6 @@ <plugin> <groupId>org.owasp</groupId> <artifactId>dependency-check-maven</artifactId> - <version>${dependency-check-maven.version}</version> <executions> <execution> <goals> diff --git a/pulsar-io/hdfs3/pom.xml b/pulsar-io/hdfs3/pom.xml index 51404dd92e3..81a7e3e2f21 100644 --- a/pulsar-io/hdfs3/pom.xml +++ b/pulsar-io/hdfs3/pom.xml @@ -27,14 +27,14 @@ </parent> <artifactId>pulsar-io-hdfs3</artifactId> <name>Pulsar IO :: Hdfs3</name> - + <dependencies> <dependency> <groupId>${project.groupId}</groupId> <artifactId>pulsar-io-core</artifactId> <version>${project.version}</version> </dependency> - + <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> @@ -49,7 +49,7 @@ <groupId>org.apache.commons</groupId> <artifactId>commons-collections4</artifactId> </dependency> - + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> @@ -80,7 +80,7 @@ </dependency> </dependencies> - + <build> <plugins> <plugin> @@ -119,7 +119,6 @@ <plugin> <groupId>org.owasp</groupId> <artifactId>dependency-check-maven</artifactId> - <version>${dependency-check-maven.version}</version> <executions> <execution> <goals> diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index 29ae765264b..b4e0c61ee26 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -205,7 +205,6 @@ <plugin> <groupId>org.owasp</groupId> <artifactId>dependency-check-maven</artifactId> - <version>${dependency-check-maven.version}</version> <executions> <execution> <goals>