| /* |
| Licensed to the Apache Software Foundation (ASF) under one or more |
| contributor license agreements. See the NOTICE file distributed with |
| this work for additional information regarding copyright ownership. |
| The ASF licenses this file to You under the Apache License, Version 2.0 |
| (the "License"); you may not use this file except in compliance with |
| the License. You may obtain a copy of the License at |
| |
| http://www.apache.org/licenses/LICENSE-2.0 |
| |
| Unless required by applicable law or agreed to in writing, software |
| distributed under the License is distributed on an "AS IS" BASIS, |
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| See the License for the specific language governing permissions and |
| limitations under the License. |
| */ |
| |
| package tasks |
| |
| import ( |
| "encoding/json" |
| "fmt" |
| "net/http" |
| "net/url" |
| "time" |
| |
| "github.com/apache/incubator-devlake/core/errors" |
| "github.com/apache/incubator-devlake/core/models/common" |
| "github.com/apache/incubator-devlake/core/plugin" |
| helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" |
| ) |
| |
| func init() { |
| RegisterSubtaskMeta(&CollectRunsMeta) |
| } |
| |
| const RAW_RUN_TABLE = "github_api_runs" |
| |
| // Although the API accepts a maximum of 100 entries per page, sometimes |
| // the response body is too large which would lead to request failures |
| // https://github.com/apache/incubator-devlake/issues/3199 |
| const PAGE_SIZE = 30 |
| |
| type SimpleGithubApiJob struct { |
| ID int64 |
| CreatedAt common.Iso8601Time `json:"created_at"` |
| } |
| |
| var CollectRunsMeta = plugin.SubTaskMeta{ |
| Name: "collectRuns", |
| EntryPoint: CollectRuns, |
| EnabledByDefault: true, |
| Description: "Collect Runs data from Github action api, supports both timeFilter and diffSync.", |
| DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, |
| DependencyTables: []string{}, |
| ProductTables: []string{RAW_RUN_TABLE}, |
| } |
| |
| func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error { |
| data := taskCtx.GetData().(*GithubTaskData) |
| collector, err := helper.NewStatefulApiCollectorForFinalizableEntity(helper.FinalizableApiCollectorArgs{ |
| RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ |
| Ctx: taskCtx, |
| Params: GithubApiParams{ |
| ConnectionId: data.Options.ConnectionId, |
| Name: data.Options.Name, |
| }, |
| Table: RAW_RUN_TABLE, |
| }, |
| ApiClient: data.ApiClient, |
| CollectNewRecordsByList: helper.FinalizableApiCollectorListArgs{ |
| PageSize: PAGE_SIZE, |
| Concurrency: 10, |
| FinalizableApiCollectorCommonArgs: helper.FinalizableApiCollectorCommonArgs{ |
| UrlTemplate: "repos/{{ .Params.Name }}/actions/runs", |
| Query: func(reqData *helper.RequestData, createdAfter *time.Time) (url.Values, errors.Error) { |
| query := url.Values{} |
| // GitHub API returns only the first 34 pages (with a size of 30) when specifying status=compleleted, try the following API request to verify the problem. |
| // https://api.github.com/repos/apache/incubator-devlake/actions/runs?per_page=30&page=35&status=completed |
| // query.Set("status", "completed") |
| query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page)) |
| query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size)) |
| return query, nil |
| }, |
| ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { |
| body := &GithubRawRunsResult{} |
| err := helper.UnmarshalResponse(res, body) |
| if err != nil { |
| return nil, err |
| } |
| if len(body.WorkflowRuns) == 0 { |
| return nil, nil |
| } |
| // filter out the runs that are not completed |
| filteredRuns := make([]json.RawMessage, 0) |
| for _, run := range body.WorkflowRuns { |
| if run.Status == "completed" { |
| runJSON, err := json.Marshal(run) |
| if err != nil { |
| return nil, errors.Convert(err) |
| } |
| filteredRuns = append(filteredRuns, json.RawMessage(runJSON)) |
| } |
| } |
| return filteredRuns, nil |
| }, |
| }, |
| GetCreated: func(item json.RawMessage) (time.Time, errors.Error) { |
| pj := &SimpleGithubApiJob{} |
| err := json.Unmarshal(item, pj) |
| if err != nil { |
| return time.Time{}, errors.BadInput.Wrap(err, "failed to unmarshal github run") |
| } |
| return pj.CreatedAt.ToTime(), nil |
| }, |
| }, |
| }) |
| |
| if err != nil { |
| return err |
| } |
| |
| return collector.Execute() |
| |
| } |
| |
| type GithubRawRunsResult struct { |
| TotalCount int `json:"total_count"` |
| WorkflowRuns []struct { |
| ID int64 `json:"id"` |
| Name string `json:"name"` |
| NodeID string `json:"node_id"` |
| HeadBranch string `json:"head_branch"` |
| HeadSha string `json:"head_sha"` |
| Path string `json:"path"` |
| DisplayTitle string `json:"display_title"` |
| RunNumber int `json:"run_number"` |
| Event string `json:"event"` |
| Status string `json:"status"` |
| Conclusion string `json:"conclusion"` |
| WorkflowID int `json:"workflow_id"` |
| CheckSuiteID int64 `json:"check_suite_id"` |
| CheckSuiteNodeID string `json:"check_suite_node_id"` |
| URL string `json:"url"` |
| HTMLURL string `json:"html_url"` |
| PullRequests []struct { |
| URL string `json:"url"` |
| ID int `json:"id"` |
| Number int `json:"number"` |
| Head struct { |
| Ref string `json:"ref"` |
| Sha string `json:"sha"` |
| Repo struct { |
| ID int `json:"id"` |
| URL string `json:"url"` |
| Name string `json:"name"` |
| } `json:"repo"` |
| } `json:"head"` |
| Base struct { |
| Ref string `json:"ref"` |
| Sha string `json:"sha"` |
| Repo struct { |
| ID int `json:"id"` |
| URL string `json:"url"` |
| Name string `json:"name"` |
| } `json:"repo"` |
| } `json:"base"` |
| } `json:"pull_requests"` |
| CreatedAt time.Time `json:"created_at"` |
| UpdatedAt time.Time `json:"updated_at"` |
| Actor struct { |
| Login string `json:"login"` |
| ID int `json:"id"` |
| NodeID string `json:"node_id"` |
| AvatarURL string `json:"avatar_url"` |
| GravatarID string `json:"gravatar_id"` |
| URL string `json:"url"` |
| HTMLURL string `json:"html_url"` |
| FollowersURL string `json:"followers_url"` |
| FollowingURL string `json:"following_url"` |
| GistsURL string `json:"gists_url"` |
| StarredURL string `json:"starred_url"` |
| SubscriptionsURL string `json:"subscriptions_url"` |
| OrganizationsURL string `json:"organizations_url"` |
| ReposURL string `json:"repos_url"` |
| EventsURL string `json:"events_url"` |
| ReceivedEventsURL string `json:"received_events_url"` |
| Type string `json:"type"` |
| SiteAdmin bool `json:"site_admin"` |
| } `json:"actor"` |
| RunAttempt int `json:"run_attempt"` |
| ReferencedWorkflows []any `json:"referenced_workflows"` |
| RunStartedAt time.Time `json:"run_started_at"` |
| TriggeringActor struct { |
| Login string `json:"login"` |
| ID int `json:"id"` |
| NodeID string `json:"node_id"` |
| AvatarURL string `json:"avatar_url"` |
| GravatarID string `json:"gravatar_id"` |
| URL string `json:"url"` |
| HTMLURL string `json:"html_url"` |
| FollowersURL string `json:"followers_url"` |
| FollowingURL string `json:"following_url"` |
| GistsURL string `json:"gists_url"` |
| StarredURL string `json:"starred_url"` |
| SubscriptionsURL string `json:"subscriptions_url"` |
| OrganizationsURL string `json:"organizations_url"` |
| ReposURL string `json:"repos_url"` |
| EventsURL string `json:"events_url"` |
| ReceivedEventsURL string `json:"received_events_url"` |
| Type string `json:"type"` |
| SiteAdmin bool `json:"site_admin"` |
| } `json:"triggering_actor"` |
| JobsURL string `json:"jobs_url"` |
| LogsURL string `json:"logs_url"` |
| CheckSuiteURL string `json:"check_suite_url"` |
| ArtifactsURL string `json:"artifacts_url"` |
| CancelURL string `json:"cancel_url"` |
| RerunURL string `json:"rerun_url"` |
| PreviousAttemptURL any `json:"previous_attempt_url"` |
| WorkflowURL string `json:"workflow_url"` |
| HeadCommit struct { |
| ID string `json:"id"` |
| TreeID string `json:"tree_id"` |
| Message string `json:"message"` |
| Timestamp time.Time `json:"timestamp"` |
| Author struct { |
| Name string `json:"name"` |
| Email string `json:"email"` |
| } `json:"author"` |
| Committer struct { |
| Name string `json:"name"` |
| Email string `json:"email"` |
| } `json:"committer"` |
| } `json:"head_commit"` |
| Repository struct { |
| ID int `json:"id"` |
| NodeID string `json:"node_id"` |
| Name string `json:"name"` |
| FullName string `json:"full_name"` |
| Private bool `json:"private"` |
| Owner struct { |
| Login string `json:"login"` |
| ID int `json:"id"` |
| NodeID string `json:"node_id"` |
| AvatarURL string `json:"avatar_url"` |
| GravatarID string `json:"gravatar_id"` |
| URL string `json:"url"` |
| HTMLURL string `json:"html_url"` |
| FollowersURL string `json:"followers_url"` |
| FollowingURL string `json:"following_url"` |
| GistsURL string `json:"gists_url"` |
| StarredURL string `json:"starred_url"` |
| SubscriptionsURL string `json:"subscriptions_url"` |
| OrganizationsURL string `json:"organizations_url"` |
| ReposURL string `json:"repos_url"` |
| EventsURL string `json:"events_url"` |
| ReceivedEventsURL string `json:"received_events_url"` |
| Type string `json:"type"` |
| SiteAdmin bool `json:"site_admin"` |
| } `json:"owner"` |
| HTMLURL string `json:"html_url"` |
| Description string `json:"description"` |
| Fork bool `json:"fork"` |
| URL string `json:"url"` |
| ForksURL string `json:"forks_url"` |
| KeysURL string `json:"keys_url"` |
| CollaboratorsURL string `json:"collaborators_url"` |
| TeamsURL string `json:"teams_url"` |
| HooksURL string `json:"hooks_url"` |
| IssueEventsURL string `json:"issue_events_url"` |
| EventsURL string `json:"events_url"` |
| AssigneesURL string `json:"assignees_url"` |
| BranchesURL string `json:"branches_url"` |
| TagsURL string `json:"tags_url"` |
| BlobsURL string `json:"blobs_url"` |
| GitTagsURL string `json:"git_tags_url"` |
| GitRefsURL string `json:"git_refs_url"` |
| TreesURL string `json:"trees_url"` |
| StatusesURL string `json:"statuses_url"` |
| LanguagesURL string `json:"languages_url"` |
| StargazersURL string `json:"stargazers_url"` |
| ContributorsURL string `json:"contributors_url"` |
| SubscribersURL string `json:"subscribers_url"` |
| SubscriptionURL string `json:"subscription_url"` |
| CommitsURL string `json:"commits_url"` |
| GitCommitsURL string `json:"git_commits_url"` |
| CommentsURL string `json:"comments_url"` |
| IssueCommentURL string `json:"issue_comment_url"` |
| ContentsURL string `json:"contents_url"` |
| CompareURL string `json:"compare_url"` |
| MergesURL string `json:"merges_url"` |
| ArchiveURL string `json:"archive_url"` |
| DownloadsURL string `json:"downloads_url"` |
| IssuesURL string `json:"issues_url"` |
| PullsURL string `json:"pulls_url"` |
| MilestonesURL string `json:"milestones_url"` |
| NotificationsURL string `json:"notifications_url"` |
| LabelsURL string `json:"labels_url"` |
| ReleasesURL string `json:"releases_url"` |
| DeploymentsURL string `json:"deployments_url"` |
| } `json:"repository"` |
| HeadRepository struct { |
| ID int `json:"id"` |
| NodeID string `json:"node_id"` |
| Name string `json:"name"` |
| FullName string `json:"full_name"` |
| Private bool `json:"private"` |
| Owner struct { |
| Login string `json:"login"` |
| ID int `json:"id"` |
| NodeID string `json:"node_id"` |
| AvatarURL string `json:"avatar_url"` |
| GravatarID string `json:"gravatar_id"` |
| URL string `json:"url"` |
| HTMLURL string `json:"html_url"` |
| FollowersURL string `json:"followers_url"` |
| FollowingURL string `json:"following_url"` |
| GistsURL string `json:"gists_url"` |
| StarredURL string `json:"starred_url"` |
| SubscriptionsURL string `json:"subscriptions_url"` |
| OrganizationsURL string `json:"organizations_url"` |
| ReposURL string `json:"repos_url"` |
| EventsURL string `json:"events_url"` |
| ReceivedEventsURL string `json:"received_events_url"` |
| Type string `json:"type"` |
| SiteAdmin bool `json:"site_admin"` |
| } `json:"owner"` |
| HTMLURL string `json:"html_url"` |
| Description string `json:"description"` |
| Fork bool `json:"fork"` |
| URL string `json:"url"` |
| ForksURL string `json:"forks_url"` |
| KeysURL string `json:"keys_url"` |
| CollaboratorsURL string `json:"collaborators_url"` |
| TeamsURL string `json:"teams_url"` |
| HooksURL string `json:"hooks_url"` |
| IssueEventsURL string `json:"issue_events_url"` |
| EventsURL string `json:"events_url"` |
| AssigneesURL string `json:"assignees_url"` |
| BranchesURL string `json:"branches_url"` |
| TagsURL string `json:"tags_url"` |
| BlobsURL string `json:"blobs_url"` |
| GitTagsURL string `json:"git_tags_url"` |
| GitRefsURL string `json:"git_refs_url"` |
| TreesURL string `json:"trees_url"` |
| StatusesURL string `json:"statuses_url"` |
| LanguagesURL string `json:"languages_url"` |
| StargazersURL string `json:"stargazers_url"` |
| ContributorsURL string `json:"contributors_url"` |
| SubscribersURL string `json:"subscribers_url"` |
| SubscriptionURL string `json:"subscription_url"` |
| CommitsURL string `json:"commits_url"` |
| GitCommitsURL string `json:"git_commits_url"` |
| CommentsURL string `json:"comments_url"` |
| IssueCommentURL string `json:"issue_comment_url"` |
| ContentsURL string `json:"contents_url"` |
| CompareURL string `json:"compare_url"` |
| MergesURL string `json:"merges_url"` |
| ArchiveURL string `json:"archive_url"` |
| DownloadsURL string `json:"downloads_url"` |
| IssuesURL string `json:"issues_url"` |
| PullsURL string `json:"pulls_url"` |
| MilestonesURL string `json:"milestones_url"` |
| NotificationsURL string `json:"notifications_url"` |
| LabelsURL string `json:"labels_url"` |
| ReleasesURL string `json:"releases_url"` |
| DeploymentsURL string `json:"deployments_url"` |
| } `json:"head_repository"` |
| } `json:"workflow_runs"` |
| } |
