This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new f71f9e969 fix: github run api page<35 limit (#6876)
f71f9e969 is described below
commit f71f9e969079d8bab3ec712cbcc5854b7e1881e7
Author: abeizn <[email protected]>
AuthorDate: Fri Jan 26 10:52:13 2024 +0800
fix: github run api page<35 limit (#6876)
* fix: github run api page<35 limit
* fix: some tips
* fix: update note
---
backend/plugins/github/tasks/cicd_run_collector.go | 270 ++++++++++++++++++++-
1 file changed, 262 insertions(+), 8 deletions(-)
diff --git a/backend/plugins/github/tasks/cicd_run_collector.go
b/backend/plugins/github/tasks/cicd_run_collector.go
index c94f194ca..3cd1bd5d5 100644
--- a/backend/plugins/github/tasks/cicd_run_collector.go
+++ b/backend/plugins/github/tasks/cicd_run_collector.go
@@ -41,11 +41,6 @@ const RAW_RUN_TABLE = "github_api_runs"
// https://github.com/apache/incubator-devlake/issues/3199
const PAGE_SIZE = 30
-type GithubRawRunsResult struct {
- TotalCount int64 `json:"total_count"`
- GithubWorkflowRuns []json.RawMessage `json:"workflow_runs"`
-}
-
type SimpleGithubApiJob struct {
ID int64
CreatedAt common.Iso8601Time `json:"created_at"`
@@ -80,7 +75,9 @@ func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error {
UrlTemplate: "repos/{{ .Params.Name
}}/actions/runs",
Query: func(reqData *helper.RequestData,
createdAfter *time.Time) (url.Values, errors.Error) {
query := url.Values{}
- query.Set("status", "completed")
+ // GitHub API returns only the first 34
pages (with a size of 30) when specifying status=compleleted, try the following
API request to verify the problem.
+ //
https://api.github.com/repos/apache/incubator-devlake/actions/runs?per_page=30&page=35&status=completed
+ // query.Set("status", "completed")
query.Set("page", fmt.Sprintf("%v",
reqData.Pager.Page))
query.Set("per_page", fmt.Sprintf("%v",
reqData.Pager.Size))
return query, nil
@@ -91,10 +88,21 @@ func CollectRuns(taskCtx plugin.SubTaskContext)
errors.Error {
if err != nil {
return nil, err
}
- if len(body.GithubWorkflowRuns) == 0 {
+ if len(body.WorkflowRuns) == 0 {
return nil, nil
}
- return body.GithubWorkflowRuns, nil
+ // filter out the runs that are not
completed
+ filteredRuns := make([]json.RawMessage,
0)
+ for _, run := range body.WorkflowRuns {
+ if run.Status == "completed" {
+ runJSON, err :=
json.Marshal(run)
+ if err != nil {
+ return nil,
errors.Convert(err)
+ }
+ filteredRuns =
append(filteredRuns, json.RawMessage(runJSON))
+ }
+ }
+ return filteredRuns, nil
},
},
GetCreated: func(item json.RawMessage) (time.Time,
errors.Error) {
@@ -115,3 +123,249 @@ func CollectRuns(taskCtx plugin.SubTaskContext)
errors.Error {
return collector.Execute()
}
+
+type GithubRawRunsResult struct {
+ TotalCount int `json:"total_count"`
+ WorkflowRuns []struct {
+ ID int64 `json:"id"`
+ Name string `json:"name"`
+ NodeID string `json:"node_id"`
+ HeadBranch string `json:"head_branch"`
+ HeadSha string `json:"head_sha"`
+ Path string `json:"path"`
+ DisplayTitle string `json:"display_title"`
+ RunNumber int `json:"run_number"`
+ Event string `json:"event"`
+ Status string `json:"status"`
+ Conclusion string `json:"conclusion"`
+ WorkflowID int `json:"workflow_id"`
+ CheckSuiteID int64 `json:"check_suite_id"`
+ CheckSuiteNodeID string `json:"check_suite_node_id"`
+ URL string `json:"url"`
+ HTMLURL string `json:"html_url"`
+ PullRequests []struct {
+ URL string `json:"url"`
+ ID int `json:"id"`
+ Number int `json:"number"`
+ Head struct {
+ Ref string `json:"ref"`
+ Sha string `json:"sha"`
+ Repo struct {
+ ID int `json:"id"`
+ URL string `json:"url"`
+ Name string `json:"name"`
+ } `json:"repo"`
+ } `json:"head"`
+ Base struct {
+ Ref string `json:"ref"`
+ Sha string `json:"sha"`
+ Repo struct {
+ ID int `json:"id"`
+ URL string `json:"url"`
+ Name string `json:"name"`
+ } `json:"repo"`
+ } `json:"base"`
+ } `json:"pull_requests"`
+ CreatedAt time.Time `json:"created_at"`
+ UpdatedAt time.Time `json:"updated_at"`
+ Actor struct {
+ Login string `json:"login"`
+ ID int `json:"id"`
+ NodeID string `json:"node_id"`
+ AvatarURL string `json:"avatar_url"`
+ GravatarID string `json:"gravatar_id"`
+ URL string `json:"url"`
+ HTMLURL string `json:"html_url"`
+ FollowersURL string `json:"followers_url"`
+ FollowingURL string `json:"following_url"`
+ GistsURL string `json:"gists_url"`
+ StarredURL string `json:"starred_url"`
+ SubscriptionsURL string `json:"subscriptions_url"`
+ OrganizationsURL string `json:"organizations_url"`
+ ReposURL string `json:"repos_url"`
+ EventsURL string `json:"events_url"`
+ ReceivedEventsURL string `json:"received_events_url"`
+ Type string `json:"type"`
+ SiteAdmin bool `json:"site_admin"`
+ } `json:"actor"`
+ RunAttempt int `json:"run_attempt"`
+ ReferencedWorkflows []any `json:"referenced_workflows"`
+ RunStartedAt time.Time `json:"run_started_at"`
+ TriggeringActor struct {
+ Login string `json:"login"`
+ ID int `json:"id"`
+ NodeID string `json:"node_id"`
+ AvatarURL string `json:"avatar_url"`
+ GravatarID string `json:"gravatar_id"`
+ URL string `json:"url"`
+ HTMLURL string `json:"html_url"`
+ FollowersURL string `json:"followers_url"`
+ FollowingURL string `json:"following_url"`
+ GistsURL string `json:"gists_url"`
+ StarredURL string `json:"starred_url"`
+ SubscriptionsURL string `json:"subscriptions_url"`
+ OrganizationsURL string `json:"organizations_url"`
+ ReposURL string `json:"repos_url"`
+ EventsURL string `json:"events_url"`
+ ReceivedEventsURL string `json:"received_events_url"`
+ Type string `json:"type"`
+ SiteAdmin bool `json:"site_admin"`
+ } `json:"triggering_actor"`
+ JobsURL string `json:"jobs_url"`
+ LogsURL string `json:"logs_url"`
+ CheckSuiteURL string `json:"check_suite_url"`
+ ArtifactsURL string `json:"artifacts_url"`
+ CancelURL string `json:"cancel_url"`
+ RerunURL string `json:"rerun_url"`
+ PreviousAttemptURL any `json:"previous_attempt_url"`
+ WorkflowURL string `json:"workflow_url"`
+ HeadCommit struct {
+ ID string `json:"id"`
+ TreeID string `json:"tree_id"`
+ Message string `json:"message"`
+ Timestamp time.Time `json:"timestamp"`
+ Author struct {
+ Name string `json:"name"`
+ Email string `json:"email"`
+ } `json:"author"`
+ Committer struct {
+ Name string `json:"name"`
+ Email string `json:"email"`
+ } `json:"committer"`
+ } `json:"head_commit"`
+ Repository struct {
+ ID int `json:"id"`
+ NodeID string `json:"node_id"`
+ Name string `json:"name"`
+ FullName string `json:"full_name"`
+ Private bool `json:"private"`
+ Owner struct {
+ Login string `json:"login"`
+ ID int `json:"id"`
+ NodeID string `json:"node_id"`
+ AvatarURL string `json:"avatar_url"`
+ GravatarID string `json:"gravatar_id"`
+ URL string `json:"url"`
+ HTMLURL string `json:"html_url"`
+ FollowersURL string `json:"followers_url"`
+ FollowingURL string `json:"following_url"`
+ GistsURL string `json:"gists_url"`
+ StarredURL string `json:"starred_url"`
+ SubscriptionsURL string
`json:"subscriptions_url"`
+ OrganizationsURL string
`json:"organizations_url"`
+ ReposURL string `json:"repos_url"`
+ EventsURL string `json:"events_url"`
+ ReceivedEventsURL string
`json:"received_events_url"`
+ Type string `json:"type"`
+ SiteAdmin bool `json:"site_admin"`
+ } `json:"owner"`
+ HTMLURL string `json:"html_url"`
+ Description string `json:"description"`
+ Fork bool `json:"fork"`
+ URL string `json:"url"`
+ ForksURL string `json:"forks_url"`
+ KeysURL string `json:"keys_url"`
+ CollaboratorsURL string `json:"collaborators_url"`
+ TeamsURL string `json:"teams_url"`
+ HooksURL string `json:"hooks_url"`
+ IssueEventsURL string `json:"issue_events_url"`
+ EventsURL string `json:"events_url"`
+ AssigneesURL string `json:"assignees_url"`
+ BranchesURL string `json:"branches_url"`
+ TagsURL string `json:"tags_url"`
+ BlobsURL string `json:"blobs_url"`
+ GitTagsURL string `json:"git_tags_url"`
+ GitRefsURL string `json:"git_refs_url"`
+ TreesURL string `json:"trees_url"`
+ StatusesURL string `json:"statuses_url"`
+ LanguagesURL string `json:"languages_url"`
+ StargazersURL string `json:"stargazers_url"`
+ ContributorsURL string `json:"contributors_url"`
+ SubscribersURL string `json:"subscribers_url"`
+ SubscriptionURL string `json:"subscription_url"`
+ CommitsURL string `json:"commits_url"`
+ GitCommitsURL string `json:"git_commits_url"`
+ CommentsURL string `json:"comments_url"`
+ IssueCommentURL string `json:"issue_comment_url"`
+ ContentsURL string `json:"contents_url"`
+ CompareURL string `json:"compare_url"`
+ MergesURL string `json:"merges_url"`
+ ArchiveURL string `json:"archive_url"`
+ DownloadsURL string `json:"downloads_url"`
+ IssuesURL string `json:"issues_url"`
+ PullsURL string `json:"pulls_url"`
+ MilestonesURL string `json:"milestones_url"`
+ NotificationsURL string `json:"notifications_url"`
+ LabelsURL string `json:"labels_url"`
+ ReleasesURL string `json:"releases_url"`
+ DeploymentsURL string `json:"deployments_url"`
+ } `json:"repository"`
+ HeadRepository struct {
+ ID int `json:"id"`
+ NodeID string `json:"node_id"`
+ Name string `json:"name"`
+ FullName string `json:"full_name"`
+ Private bool `json:"private"`
+ Owner struct {
+ Login string `json:"login"`
+ ID int `json:"id"`
+ NodeID string `json:"node_id"`
+ AvatarURL string `json:"avatar_url"`
+ GravatarID string `json:"gravatar_id"`
+ URL string `json:"url"`
+ HTMLURL string `json:"html_url"`
+ FollowersURL string `json:"followers_url"`
+ FollowingURL string `json:"following_url"`
+ GistsURL string `json:"gists_url"`
+ StarredURL string `json:"starred_url"`
+ SubscriptionsURL string
`json:"subscriptions_url"`
+ OrganizationsURL string
`json:"organizations_url"`
+ ReposURL string `json:"repos_url"`
+ EventsURL string `json:"events_url"`
+ ReceivedEventsURL string
`json:"received_events_url"`
+ Type string `json:"type"`
+ SiteAdmin bool `json:"site_admin"`
+ } `json:"owner"`
+ HTMLURL string `json:"html_url"`
+ Description string `json:"description"`
+ Fork bool `json:"fork"`
+ URL string `json:"url"`
+ ForksURL string `json:"forks_url"`
+ KeysURL string `json:"keys_url"`
+ CollaboratorsURL string `json:"collaborators_url"`
+ TeamsURL string `json:"teams_url"`
+ HooksURL string `json:"hooks_url"`
+ IssueEventsURL string `json:"issue_events_url"`
+ EventsURL string `json:"events_url"`
+ AssigneesURL string `json:"assignees_url"`
+ BranchesURL string `json:"branches_url"`
+ TagsURL string `json:"tags_url"`
+ BlobsURL string `json:"blobs_url"`
+ GitTagsURL string `json:"git_tags_url"`
+ GitRefsURL string `json:"git_refs_url"`
+ TreesURL string `json:"trees_url"`
+ StatusesURL string `json:"statuses_url"`
+ LanguagesURL string `json:"languages_url"`
+ StargazersURL string `json:"stargazers_url"`
+ ContributorsURL string `json:"contributors_url"`
+ SubscribersURL string `json:"subscribers_url"`
+ SubscriptionURL string `json:"subscription_url"`
+ CommitsURL string `json:"commits_url"`
+ GitCommitsURL string `json:"git_commits_url"`
+ CommentsURL string `json:"comments_url"`
+ IssueCommentURL string `json:"issue_comment_url"`
+ ContentsURL string `json:"contents_url"`
+ CompareURL string `json:"compare_url"`
+ MergesURL string `json:"merges_url"`
+ ArchiveURL string `json:"archive_url"`
+ DownloadsURL string `json:"downloads_url"`
+ IssuesURL string `json:"issues_url"`
+ PullsURL string `json:"pulls_url"`
+ MilestonesURL string `json:"milestones_url"`
+ NotificationsURL string `json:"notifications_url"`
+ LabelsURL string `json:"labels_url"`
+ ReleasesURL string `json:"releases_url"`
+ DeploymentsURL string `json:"deployments_url"`
+ } `json:"head_repository"`
+ } `json:"workflow_runs"`
+}
