This is an automated email from the ASF dual-hosted git repository.
snazy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new 81ad20639 CI: Address failure from accessing GH API (#2132)
81ad20639 is described below
commit 81ad20639d9aceb3a0b5bbe1dc1eabff56177ae2
Author: Robert Stupp <[email protected]>
AuthorDate: Mon Jul 21 09:52:32 2025 +0200
CI: Address failure from accessing GH API (#2132)
CI sometimes fails with this failure:
```
* What went wrong:
Execution failed for task ':generatePomFileForMavenPublication'.
> Unable to process url:
https://api.github.com/repos/apache/polaris/contributors?per_page=1000
```
The sometimes failing request fetches the list of contributors to be
published in the "root" POM. Unauthorized GH API requests have an hourly(?)
limit of 60 requests per source IP. Authorized requests have a much higher rate
limit. We do have a GitHub token available in every CI run, which can be used
in GH API requests. This change adds the `Authorization` header for the failing
GH API request to leverage the higher rate limit and let CI not fail (that
often).
---
.github/workflows/gradle.yml | 3 +++
.github/workflows/nightly.yml | 2 ++
.github/workflows/spark_client_regtests.yml | 3 +++
build-logic/src/main/kotlin/publishing/util.kt | 22 ++++++++++++++++++++--
4 files changed, 28 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
index b8f4acca2..a2284fa84 100644
--- a/.github/workflows/gradle.yml
+++ b/.github/workflows/gradle.yml
@@ -53,6 +53,9 @@ jobs:
- name: Prepare Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-prepare
- name: Run unit tests
+ env:
+ # publishToMavenLocal causes a GH API requests, use the token for
those requests
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
./gradlew check sourceTarball distTar distZip publishToMavenLocal \
-x :polaris-runtime-service:test \
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index 5af7bfb42..8ed71ef62 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -59,5 +59,7 @@ jobs:
- name: Publish SNAPSHOTs to Apache Nexus Repository
run: ./gradlew publishToApache
env:
+ # publishToApache causes a GH API requests, use the token for those
requests
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ORG_GRADLE_PROJECT_apacheUsername: ${{ secrets.NEXUS_USER }}
ORG_GRADLE_PROJECT_apachePassword: ${{ secrets.NEXUS_PW }}
diff --git a/.github/workflows/spark_client_regtests.yml
b/.github/workflows/spark_client_regtests.yml
index 016d39fec..11b10a50a 100644
--- a/.github/workflows/spark_client_regtests.yml
+++ b/.github/workflows/spark_client_regtests.yml
@@ -55,6 +55,9 @@ jobs:
run: ./gradlew regeneratePythonClient
- name: Project build without testing
+ env:
+ # publishToMavenLocal causes a GH API requests, use the token for
those requests
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: ./gradlew assemble publishToMavenLocal
- name: Image build
diff --git a/build-logic/src/main/kotlin/publishing/util.kt
b/build-logic/src/main/kotlin/publishing/util.kt
index 2f073d4d7..864a14c72 100644
--- a/build-logic/src/main/kotlin/publishing/util.kt
+++ b/build-logic/src/main/kotlin/publishing/util.kt
@@ -61,11 +61,29 @@ internal fun <T : Any> unsafeCast(o: Any?): T {
return o as T
}
-internal fun <T : Any> parseJson(url: String): T {
+internal fun <T : Any> parseJson(urlStr: String): T {
+ val url = URI(urlStr).toURL()
+
+ val headers = mutableMapOf<String, String>()
+ if (url.host == "api.github.com" && url.protocol == "https") {
+ val githubToken = System.getenv("GITHUB_TOKEN")
+ if (githubToken != null) {
+ // leverage the GH token to benefit from higher rate limits
+ headers["Authorization"] = "Bearer $githubToken"
+ // recommended for GH API requests
+ headers["X-GitHub-Api-Version"] = "2022-11-28"
+ headers["Accept"] = "application/vnd.github+json"
+ }
+ }
+
+ // See org.codehaus.groovy.runtime.ResourceGroovyMethods.newReader(URL, Map)
+ val params = mapOf("requestProperties" to headers)
+
+ val slurper = JsonSlurper()
var attempt = 0
while (true) {
try {
- return unsafeCast(JsonSlurper().parse(URI(url).toURL())) as T
+ return unsafeCast(slurper.parse(params, url)) as T
} catch (e: JsonException) {
if (e.cause is FileNotFoundException) {
throw e
