This is an automated email from the ASF dual-hosted git repository. lhotari pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/branch-3.2 by this push: new 3a125c4169c [fix][ci] Fix OWASP Dependency Check download by using NVD API key (#22999) 3a125c4169c is described below commit 3a125c4169cbb55f6f05e50e3bd722609faf3bb2 Author: Lari Hotari <lhot...@users.noreply.github.com> AuthorDate: Thu Jul 4 12:41:21 2024 +0300 [fix][ci] Fix OWASP Dependency Check download by using NVD API key (#22999) (cherry picked from commit 8b7754f11f113af9d341a460795d0c7b8095f594) # Conflicts: # .github/workflows/ci-owasp-dependency-check.yaml # pom.xml --- .github/workflows/ci-owasp-dependency-check.yaml | 80 +++++++++++++++++------- .github/workflows/pulsar-ci.yaml | 7 +-- distribution/io/pom.xml | 1 - pom.xml | 14 ++++- pulsar-io/docs/pom.xml | 1 - pulsar-io/flume/pom.xml | 1 - pulsar-io/hbase/pom.xml | 1 - pulsar-io/hdfs2/pom.xml | 7 +-- pulsar-io/hdfs3/pom.xml | 9 ++- tiered-storage/file-system/pom.xml | 1 - 10 files changed, 79 insertions(+), 43 deletions(-) diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 0ee1275bdfe..a70f4a82ff1 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -24,7 +24,9 @@ on: workflow_dispatch: env: - MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + JDK_DISTRIBUTION: corretto + NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }} jobs: run-owasp-dependency-check: @@ -34,62 +36,96 @@ jobs: JOB_NAME: Check ${{ matrix.branch }} GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} runs-on: ubuntu-22.04 - timeout-minutes: 45 + timeout-minutes: 75 strategy: fail-fast: false + max-parallel: 1 matrix: include: - branch: master - - branch: branch-3.1 + - branch: branch-3.3 + - branch: branch-3.2 - branch: branch-3.0 - - branch: branch-2.11 - - branch: branch-2.10 - jdk: 11 - - branch: branch-2.9 - jdk: 11 - - branch: branch-2.8 - jdk: 11 steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ matrix.branch }} - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Cache local Maven repository - uses: actions/cache@v3 + - name: Restore Maven repository cache + uses: actions/cache/restore@v4 timeout-minutes: 5 with: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-owasp-${{ hashFiles('**/pom.xml') }} + key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} restore-keys: | - ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK ${{ matrix.jdk || '17' }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: - distribution: 'temurin' + distribution: ${{ env.JDK_DISTRIBUTION }} java-version: ${{ matrix.jdk || '17' }} - name: run install by skip tests - run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true + run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true -DnarPluginPhase=none -pl '!distribution/io,!distribution/offloaders' + + - name: OWASP cache key weeknum + id: get-weeknum + run: | + echo "weeknum=$(date -u +"%Y-%U")" >> $GITHUB_OUTPUT + shell: bash + + - name: Restore OWASP Dependency Check data + id: restore-owasp-dependency-check-data + uses: actions/cache/restore@v4 + timeout-minutes: 5 + with: + path: ~/.m2/repository/org/owasp/dependency-check-data + key: owasp-dependency-check-data-${{ steps.get-weeknum.outputs.weeknum }} + enableCrossOsArchive: true + restore-keys: | + owasp-dependency-check-data- + + - name: Update OWASP Dependency Check data + id: update-owasp-dependency-check-data + if: ${{ matrix.branch == 'master' && (steps.restore-owasp-dependency-check-data.outputs.cache-hit != 'true' || steps.restore-owasp-dependency-check-data.outputs.cache-matched-key != steps.restore-owasp-dependency-check-data.outputs.cache-primary-key) }} + run: mvn -B -ntp -Powasp-dependency-check initialize -pl . dependency-check:update-only + + - name: Save OWASP Dependency Check data + if: ${{ steps.update-owasp-dependency-check-data.outcome == 'success' }} + uses: actions/cache/save@v4 + timeout-minutes: 5 + with: + path: ~/.m2/repository/org/owasp/dependency-check-data + key: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-primary-key }} + enableCrossOsArchive: true - name: run OWASP Dependency Check for distribution/server (-DfailBuildOnAnyVulnerability=true) run: mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl distribution/server -DfailBuildOnAnyVulnerability=true + - name: run OWASP Dependency Check for offloaders/tiered-storage and pulsar-io connectors (-DfailOnError=false) + if: ${{ !cancelled() }} + run: | + mvnprojects=$(mvn -B -ntp -Dscan=false initialize \ + | grep -- "-< .* >-" \ + | sed -E 's/.*-< (.*) >-.*/\1/' \ + | grep -E 'pulsar-io-|tiered-storage-|offloader' \ + | tr '\n' ',' | sed 's/,$/\n/' ) + set -xe + mvn --fail-at-end -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -DfailOnError=false -pl "${mvnprojects}" + - name: Upload OWASP Dependency Check reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: owasp-dependency-check-reports-${{ matrix.branch }} path: | - distribution/server/target/dependency-check-report.html - distribution/offloaders/target/dependency-check-report.html - distribution/io/target/dependency-check-report.html + **/target/dependency-check-report.html \ No newline at end of file diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 7cee6c3db3d..2a9a58c08d0 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -1338,6 +1338,7 @@ jobs: env: GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} + NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }} steps: - name: checkout uses: actions/checkout@v4 @@ -1353,16 +1354,14 @@ jobs: with: limit-access-to-actor: true - - name: Cache Maven dependencies - uses: actions/cache@v4 + - name: Restore Maven repository cache + uses: actions/cache/restore@v4 timeout-minutes: 5 with: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar - !~/.m2/repository/org/owasp/dependency-check-data key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - lookup-only: true restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index bab6df3441c..61abe5670bc 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -137,7 +137,6 @@ <plugin> <groupId>org.owasp</groupId> <artifactId>dependency-check-maven</artifactId> - <version>${dependency-check-maven.version}</version> <executions> <execution> <goals> diff --git a/pom.xml b/pom.xml index a8fa4bd6d0b..55fd073ddb5 100644 --- a/pom.xml +++ b/pom.xml @@ -305,7 +305,7 @@ flexible messaging model and an intuitive client API.</description> <errorprone-slf4j.version>0.1.4</errorprone-slf4j.version> <j2objc-annotations.version>1.3</j2objc-annotations.version> <lightproto-maven-plugin.version>0.4</lightproto-maven-plugin.version> - <dependency-check-maven.version>9.1.0</dependency-check-maven.version> + <dependency-check-maven.version>10.0.1</dependency-check-maven.version> <roaringbitmap.version>0.9.44</roaringbitmap.version> <extra-enforcer-rules.version>1.6.1</extra-enforcer-rules.version> <oshi.version>6.4.0</oshi.version> @@ -2090,6 +2090,16 @@ flexible messaging model and an intuitive client API.</description> <artifactId>build-helper-maven-plugin</artifactId> <version>${build-helper-maven-plugin.version}</version> </plugin> + <plugin> + <groupId>org.owasp</groupId> + <artifactId>dependency-check-maven</artifactId> + <version>${dependency-check-maven.version}</version> + <configuration> + <nvdApiKeyEnvironmentVariable>NIST_NVD_API_KEY</nvdApiKeyEnvironmentVariable> + <!-- Uncomment the following to use the NVD data feed provided by the Dependency-Check project --> + <!-- <nvdDatafeedUrl>https://jeremylong.github.io/DependencyCheck/hb_nvd/</nvdDatafeedUrl> --> + </configuration> + </plugin> </plugins> </pluginManagement> <extensions> @@ -2526,7 +2536,6 @@ flexible messaging model and an intuitive client API.</description> <plugin> <groupId>org.owasp</groupId> <artifactId>dependency-check-maven</artifactId> - <version>${dependency-check-maven.version}</version> <configuration> <suppressionFiles> <suppressionFile>${pulsar.basedir}/src/owasp-dependency-check-false-positives.xml</suppressionFile> @@ -2561,7 +2570,6 @@ flexible messaging model and an intuitive client API.</description> <plugin> <groupId>org.owasp</groupId> <artifactId>dependency-check-maven</artifactId> - <version>${dependency-check-maven.version}</version> <reportSets> <reportSet> <reports> diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml index ff23e090595..4100b08a272 100644 --- a/pulsar-io/docs/pom.xml +++ b/pulsar-io/docs/pom.xml @@ -253,7 +253,6 @@ <plugin> <groupId>org.owasp</groupId> <artifactId>dependency-check-maven</artifactId> - <version>${dependency-check-maven.version}</version> <executions> <execution> <goals> diff --git a/pulsar-io/flume/pom.xml b/pulsar-io/flume/pom.xml index f2a5e712c00..bcd62b412c1 100644 --- a/pulsar-io/flume/pom.xml +++ b/pulsar-io/flume/pom.xml @@ -141,7 +141,6 @@ <plugin> <groupId>org.owasp</groupId> <artifactId>dependency-check-maven</artifactId> - <version>${dependency-check-maven.version}</version> <executions> <execution> <goals> diff --git a/pulsar-io/hbase/pom.xml b/pulsar-io/hbase/pom.xml index 39986667bc1..88a7d22dfaa 100644 --- a/pulsar-io/hbase/pom.xml +++ b/pulsar-io/hbase/pom.xml @@ -108,7 +108,6 @@ <plugin> <groupId>org.owasp</groupId> <artifactId>dependency-check-maven</artifactId> - <version>${dependency-check-maven.version}</version> <executions> <execution> <goals> diff --git a/pulsar-io/hdfs2/pom.xml b/pulsar-io/hdfs2/pom.xml index b613f703205..5c7cfb9d3eb 100644 --- a/pulsar-io/hdfs2/pom.xml +++ b/pulsar-io/hdfs2/pom.xml @@ -27,14 +27,14 @@ </parent> <artifactId>pulsar-io-hdfs2</artifactId> <name>Pulsar IO :: Hdfs2</name> - + <dependencies> <dependency> <groupId>${project.groupId}</groupId> <artifactId>pulsar-io-core</artifactId> <version>${project.version}</version> </dependency> - + <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> @@ -74,7 +74,7 @@ <artifactId>commons-lang3</artifactId> </dependency> </dependencies> - + <build> <plugins> <plugin> @@ -113,7 +113,6 @@ <plugin> <groupId>org.owasp</groupId> <artifactId>dependency-check-maven</artifactId> - <version>${dependency-check-maven.version}</version> <executions> <execution> <goals> diff --git a/pulsar-io/hdfs3/pom.xml b/pulsar-io/hdfs3/pom.xml index 00703154a61..547e29e9864 100644 --- a/pulsar-io/hdfs3/pom.xml +++ b/pulsar-io/hdfs3/pom.xml @@ -27,14 +27,14 @@ </parent> <artifactId>pulsar-io-hdfs3</artifactId> <name>Pulsar IO :: Hdfs3</name> - + <dependencies> <dependency> <groupId>${project.groupId}</groupId> <artifactId>pulsar-io-core</artifactId> <version>${project.version}</version> </dependency> - + <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> @@ -49,7 +49,7 @@ <groupId>org.apache.commons</groupId> <artifactId>commons-collections4</artifactId> </dependency> - + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> @@ -80,7 +80,7 @@ </dependency> </dependencies> - + <build> <plugins> <plugin> @@ -119,7 +119,6 @@ <plugin> <groupId>org.owasp</groupId> <artifactId>dependency-check-maven</artifactId> - <version>${dependency-check-maven.version}</version> <executions> <execution> <goals> diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index ed637860e1d..5b0d6ed34ef 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -205,7 +205,6 @@ <plugin> <groupId>org.owasp</groupId> <artifactId>dependency-check-maven</artifactId> - <version>${dependency-check-maven.version}</version> <executions> <execution> <goals>