This is an automated email from the ASF dual-hosted git repository.
sjaranowski pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/maven-dist-tool.git
The following commit(s) were added to refs/heads/master by this push:
new 92ca72d Replaces HTML scrapping with plain Git to get list of branches
92ca72d is described below
commit 92ca72dfbea67b31fe04b2292ece1dcc93594d56
Author: Filipe Roque <[email protected]>
AuthorDate: Fri Apr 25 11:15:15 2025 +0100
Replaces HTML scrapping with plain Git to get list of branches
---
pom.xml | 5 ++
.../maven/dist/tools/jobs/AbstractJobsReport.java | 9 +-
.../dist/tools/jobs/branches/BranchesResponse.java | 96 ----------------------
.../tools/jobs/branches/ListBranchesReport.java | 89 ++++++++++----------
4 files changed, 55 insertions(+), 144 deletions(-)
diff --git a/pom.xml b/pom.xml
index 32b549c..65cc0df 100644
--- a/pom.xml
+++ b/pom.xml
@@ -153,6 +153,11 @@
<artifactId>github-api</artifactId>
<version>1.129</version>
</dependency>
+ <dependency>
+ <groupId>org.eclipse.jgit</groupId>
+ <artifactId>org.eclipse.jgit</artifactId>
+ <version>7.2.0.202503040940-r</version>
+ </dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
diff --git
a/src/main/java/org/apache/maven/dist/tools/jobs/AbstractJobsReport.java
b/src/main/java/org/apache/maven/dist/tools/jobs/AbstractJobsReport.java
index 49cc1a5..3b5bd9d 100644
--- a/src/main/java/org/apache/maven/dist/tools/jobs/AbstractJobsReport.java
+++ b/src/main/java/org/apache/maven/dist/tools/jobs/AbstractJobsReport.java
@@ -39,14 +39,17 @@ public abstract class AbstractJobsReport extends
AbstractMavenReport {
private static final Collection<String> EXCLUDED = Arrays.asList(
"maven-blog",
+ "maven-build-helper-plugin",
+ "maven-gh-actions-shared",
+ "maven-hocon-extension",
"maven-integration-testing", // runs with Maven core job
"maven-jenkins-env",
"maven-jenkins-lib",
+ "maven-metric-extension",
+ "maven-mvnd",
"maven-sources",
"maven-studies",
- "maven-mvnd",
- "maven-metric-extension",
- "maven-gh-actions-shared");
+ "maven-xinclude-extension");
/**
* Extract Git repository names for Apache Maven from
diff --git
a/src/main/java/org/apache/maven/dist/tools/jobs/branches/BranchesResponse.java
b/src/main/java/org/apache/maven/dist/tools/jobs/branches/BranchesResponse.java
deleted file mode 100644
index d4c154d..0000000
---
a/src/main/java/org/apache/maven/dist/tools/jobs/branches/BranchesResponse.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.maven.dist.tools.jobs.branches;
-
-import java.util.List;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-public class BranchesResponse {
-
- @JsonProperty("payload")
- private Payload payload;
-
- public Payload getPayload() {
- return payload;
- }
-
- public void setPayload(final Payload payload) {
- this.payload = payload;
- }
-
- public static class Payload {
- @JsonProperty("current_page")
- private int currentPage;
-
- @JsonProperty("has_more")
- private boolean hasMore;
-
- @JsonProperty("per_page")
- private int perPage;
-
- @JsonProperty("branches")
- private List<Branch> branches;
-
- public int getCurrentPage() {
- return currentPage;
- }
-
- public void setCurrentPage(final int currentPage) {
- this.currentPage = currentPage;
- }
-
- public boolean hasMore() {
- return hasMore;
- }
-
- public void setHasMore(final boolean hasMore) {
- this.hasMore = hasMore;
- }
-
- public int getPerPage() {
- return perPage;
- }
-
- public void setPerPage(final int perPage) {
- this.perPage = perPage;
- }
-
- public List<Branch> getBranches() {
- return branches;
- }
-
- public void setBranches(final List<Branch> branches) {
- this.branches = branches;
- }
- }
-
- public static class Branch {
- @JsonProperty("name")
- private String name;
-
- public String getName() {
- return name;
- }
-
- public void setName(final String name) {
- this.name = name;
- }
- }
-}
diff --git
a/src/main/java/org/apache/maven/dist/tools/jobs/branches/ListBranchesReport.java
b/src/main/java/org/apache/maven/dist/tools/jobs/branches/ListBranchesReport.java
index e929cf0..43743ac 100644
---
a/src/main/java/org/apache/maven/dist/tools/jobs/branches/ListBranchesReport.java
+++
b/src/main/java/org/apache/maven/dist/tools/jobs/branches/ListBranchesReport.java
@@ -32,16 +32,16 @@ import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.maven.dist.tools.JsoupRetry;
import org.apache.maven.dist.tools.jobs.AbstractJobsReport;
-import org.apache.maven.dist.tools.jobs.branches.BranchesResponse.Branch;
import org.apache.maven.doxia.sink.Sink;
import org.apache.maven.doxia.sink.SinkEventAttributes;
import org.apache.maven.doxia.sink.impl.SinkEventAttributeSet;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.reporting.MavenReportException;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.lib.Ref;
import org.jsoup.nodes.Document;
/**
@@ -191,7 +191,6 @@ public class ListBranchesReport extends AbstractJobsReport {
try {
Document jenkinsBranchesDoc = JsoupRetry.get(repositoryJobUrl);
- BranchesResponse branchesResponse = null;
Result result = new Result(repository, repositoryJobUrl);
int masterBranchesGit = 0;
int masterBranchesJenkins = 0;
@@ -201,45 +200,32 @@ public class ListBranchesReport extends
AbstractJobsReport {
Collection<String> dependabotBranchesJenkins = new
ArrayList<>();
Collection<String> restGit = new ArrayList<>();
Collection<String> restJenkins = new ArrayList<>();
- int page = 1;
- do {
- final String githubBranchesUrl =
getGitHubBranchesUrl(repository, page);
- Document githubBranchesDoc =
JsoupRetry.get(githubBranchesUrl);
- ObjectMapper objectMapper = new ObjectMapper();
-
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES,
false);
- branchesResponse =
-
objectMapper.readValue(githubBranchesDoc.body().text(), BranchesResponse.class);
+ for (String branch : getBranches(repository)) {
+ if ("master".equals(branch)) {
+ masterBranchesGit++;
- for (Branch branch :
branchesResponse.getPayload().getBranches()) {
- String name = branch.getName();
-
- if ("master".equals(name)) {
- masterBranchesGit++;
-
- if
(jenkinsBranchesDoc.getElementById("job_master") != null) {
- masterBranchesJenkins++;
- }
- } else if (JIRAPROJECTS.containsKey(repository)
- &&
name.toUpperCase().startsWith(JIRAPROJECTS.get(repository) + '-')) {
- jiraBranchesGit.add(name);
- if
(jenkinsBranchesDoc.getElementById(URLEncoder.encode("job_" + name, "UTF-8"))
!= null) {
- jiraBranchesJenkins.add(name);
- }
- } else if (name.startsWith("dependabot/")) {
- dependabotBranchesGit.add(name);
- if
(jenkinsBranchesDoc.getElementById(URLEncoder.encode("job_" + name, "UTF-8"))
!= null) {
- dependabotBranchesJenkins.add(name);
- }
- } else {
- restGit.add(name);
- if
(jenkinsBranchesDoc.getElementById(URLEncoder.encode("job_" + name, "UTF-8"))
!= null) {
- restJenkins.add(name);
- }
+ if (jenkinsBranchesDoc.getElementById("job_master") !=
null) {
+ masterBranchesJenkins++;
+ }
+ } else if (JIRAPROJECTS.containsKey(repository)
+ &&
branch.toUpperCase().startsWith(JIRAPROJECTS.get(repository) + '-')) {
+ jiraBranchesGit.add(branch);
+ if
(jenkinsBranchesDoc.getElementById(URLEncoder.encode("job_" + branch, "UTF-8"))
!= null) {
+ jiraBranchesJenkins.add(branch);
+ }
+ } else if (branch.startsWith("dependabot/")) {
+ dependabotBranchesGit.add(branch);
+ if
(jenkinsBranchesDoc.getElementById(URLEncoder.encode("job_" + branch, "UTF-8"))
!= null) {
+ dependabotBranchesJenkins.add(branch);
+ }
+ } else {
+ restGit.add(branch);
+ if
(jenkinsBranchesDoc.getElementById(URLEncoder.encode("job_" + branch, "UTF-8"))
!= null) {
+ restJenkins.add(branch);
}
}
- page = branchesResponse.getPayload().getCurrentPage() + 1;
- } while (branchesResponse.getPayload().hasMore());
+ }
result.setMasterBranchesGit(masterBranchesGit);
result.setMasterBranchesJenkins(masterBranchesJenkins);
@@ -251,7 +237,7 @@ public class ListBranchesReport extends AbstractJobsReport {
result.setRestJenkins(restJenkins);
repoStatus.add(result);
- } catch (IOException e) {
+ } catch (IOException | GitAPIException e) {
getLog().warn("Failed to read status for " + repository + "
Jenkins job " + repositoryJobUrl);
}
}
@@ -263,10 +249,6 @@ public class ListBranchesReport extends AbstractJobsReport
{
return GITHUB_URL + repository + "/branches/all";
}
- private String getGitHubBranchesUrl(String repository, int page) {
- return GITHUB_URL + repository + "/branches/all?page=" + page;
- }
-
@SuppressWarnings("checkstyle:MethodLength")
private void generateReport(List<Result> repoStatus) {
AtomicInteger masterJenkinsTotal = new AtomicInteger();
@@ -289,7 +271,7 @@ public class ListBranchesReport extends AbstractJobsReport {
sink.body();
sink.paragraph();
sink.rawText("Values are shown as <code>jenkinsBranches /
gitBranches</code>, "
- + "because not all branches end up in Jenkins, this depends on
the existence of the JenkinsFile.<br>");
+ + "because not all branches end up in Jenkins, this depends on
the existence of the Jenkinsfile.<br>");
sink.rawText("Hover over the values to see branch names, values link
to its URL to Jenkins or Gitbox</br>");
sink.rawText("For Dependabot an empty field means there's no <code>" +
DEPENDABOT_CONFIG + "</code>");
@@ -519,4 +501,21 @@ public class ListBranchesReport extends AbstractJobsReport
{
return con.getResponseCode() == HttpURLConnection.HTTP_OK;
}
+
+ private Collection<String> getBranches(String repository) throws
GitAPIException {
+ final var refs = Git.lsRemoteRepository()
+ .setHeads(true)
+ .setTags(false)
+ .setRemote(GITHUB_URL + repository + ".git")
+ .call();
+
+ final var branches = new ArrayList<String>();
+ for (final Ref ref : refs) {
+ final var name = ref.getName();
+ var branch = name.substring(11);
+ branches.add(branch);
+ System.out.println(branch);
+ }
+ return branches;
+ }
}
