Merge pull request #1472 from merico-dev/jc-1462-config-ui-blueprints
`feat` config-ui : implement pipeline blueprints
diff --git a/models/domainlayer/code/pull_request.go b/models/domainlayer/code/pull_request.go
index 67e4922..68ac6fe 100644
--- a/models/domainlayer/code/pull_request.go
+++ b/models/domainlayer/code/pull_request.go
@@ -16,7 +16,7 @@
Description string
Url string `gorm:"type:char(255)"`
AuthorName string `gorm:"type:char(100)"`
- AuthorId int
+ AuthorId string `gorm:"type:char(100)"`
ParentPrId string `gorm:"index;type:varchar(100)"`
Key int
CreatedDate time.Time
diff --git a/models/domainlayer/crossdomain/pull_request_issue.go b/models/domainlayer/crossdomain/pull_request_issue.go
index 6560430..2c885f5 100644
--- a/models/domainlayer/crossdomain/pull_request_issue.go
+++ b/models/domainlayer/crossdomain/pull_request_issue.go
@@ -3,9 +3,9 @@
import "github.com/merico-dev/lake/models/common"
type PullRequestIssue struct {
- PullRequestId string `json:"id" gorm:"primaryKey;type:varchar(255);comment:This key is generated based on details from the original plugin"` // format: <Plugin>:<Entity>:<PK0>:<PK1>
- IssueId string `gorm:"primaryKey;type:varchar(255)"`
- PullNumber int
- IssueNumber int
+ PullRequestId string `json:"id" gorm:"primaryKey;type:varchar(255);comment:This key is generated based on details from the original plugin"` // format: <Plugin>:<Entity>:<PK0>:<PK1>
+ IssueId string `gorm:"primaryKey;type:varchar(255)"`
+ PullRequestNumber int
+ IssueNumber int
common.NoPKModel
}
diff --git a/models/migrationscripts/archived/pull_request.go b/models/migrationscripts/archived/pull_request.go
index 08e4c32..3c92619 100644
--- a/models/migrationscripts/archived/pull_request.go
+++ b/models/migrationscripts/archived/pull_request.go
@@ -14,7 +14,7 @@
Description string
Url string `gorm:"type:char(255)"`
AuthorName string `gorm:"type:char(100)"`
- AuthorId int
+ AuthorId string `gorm:"type:char(100)"`
ParentPrId string `gorm:"index;type:varchar(100)"`
Key int
CreatedDate time.Time
@@ -36,10 +36,10 @@
}
type PullRequestIssue struct {
- PullRequestId string `json:"id" gorm:"primaryKey;type:varchar(255);comment:This key is generated based on details from the original plugin"` // format: <Plugin>:<Entity>:<PK0>:<PK1>
- IssueId string `gorm:"primaryKey;type:varchar(255)"`
- PullNumber int
- IssueNumber int
+ PullRequestId string `json:"id" gorm:"primaryKey;type:varchar(255);comment:This key is generated based on details from the original plugin"` // format: <Plugin>:<Entity>:<PK0>:<PK1>
+ IssueId string `gorm:"primaryKey;type:varchar(255)"`
+ PullRequestNumber int
+ IssueNumber int
NoPKModel
}
diff --git a/plugins/github/models/github_pull_request_issue.go b/plugins/github/models/github_pull_request_issue.go
index eb9149b..8745252 100644
--- a/plugins/github/models/github_pull_request_issue.go
+++ b/plugins/github/models/github_pull_request_issue.go
@@ -3,14 +3,13 @@
import "github.com/merico-dev/lake/models/common"
type GithubPullRequestIssue struct {
- PullRequestId int `gorm:"primaryKey"`
- IssueId int `gorm:"primaryKey"`
- PullNumber int
- IssueNumber int
+ PullRequestId int `gorm:"primaryKey"`
+ IssueId int `gorm:"primaryKey"`
+ PullRequestNumber int
+ IssueNumber int
common.NoPKModel
}
-func (GithubPullRequestIssue) TableName() string{
+func (GithubPullRequestIssue) TableName() string {
return "_tool_github_pull_request_issues"
}
-
diff --git a/plugins/github/models/migrationscripts/archived/pull_request.go b/plugins/github/models/migrationscripts/archived/pull_request.go
index 8b8188d..2945ae1 100644
--- a/plugins/github/models/migrationscripts/archived/pull_request.go
+++ b/plugins/github/models/migrationscripts/archived/pull_request.go
@@ -33,7 +33,7 @@
HeadCommitSha string `gorm:"type:varchar(255)"`
Url string `gorm:"type:char(255)"`
AuthorName string `gorm:"type:char(100)"`
- AuthorId int
+ AuthorId string `gorm:"type:char(100)"`
archived.NoPKModel
}
diff --git a/plugins/github/models/migrationscripts/archived/pull_request_issue.go b/plugins/github/models/migrationscripts/archived/pull_request_issue.go
index efa7209..46b191c 100644
--- a/plugins/github/models/migrationscripts/archived/pull_request_issue.go
+++ b/plugins/github/models/migrationscripts/archived/pull_request_issue.go
@@ -3,10 +3,10 @@
import "github.com/merico-dev/lake/models/migrationscripts/archived"
type GithubPullRequestIssue struct {
- PullRequestId int `gorm:"primaryKey"`
- IssueId int `gorm:"primaryKey"`
- PullNumber int
- IssueNumber int
+ PullRequestId int `gorm:"primaryKey"`
+ IssueId int `gorm:"primaryKey"`
+ PullRequestNumber int
+ IssueNumber int
archived.NoPKModel
}
diff --git a/plugins/github/tasks/pull_request_converter.go b/plugins/github/tasks/pull_request_converter.go
index aeecd8e..2801f47 100644
--- a/plugins/github/tasks/pull_request_converter.go
+++ b/plugins/github/tasks/pull_request_converter.go
@@ -7,7 +7,7 @@
"github.com/merico-dev/lake/models/domainlayer/code"
"github.com/merico-dev/lake/models/domainlayer/didgen"
"github.com/merico-dev/lake/plugins/core"
- githubModels "github.com/merico-dev/lake/plugins/github/models"
+ "github.com/merico-dev/lake/plugins/github/models"
"github.com/merico-dev/lake/plugins/helper"
)
@@ -23,17 +23,18 @@
data := taskCtx.GetData().(*GithubTaskData)
repoId := data.Repo.GithubId
- cursor, err := db.Model(&githubModels.GithubPullRequest{}).Where("repo_id = ?", repoId).Rows()
+ cursor, err := db.Model(&models.GithubPullRequest{}).Where("repo_id = ?", repoId).Rows()
if err != nil {
return err
}
defer cursor.Close()
- prIdGen := didgen.NewDomainIdGenerator(&githubModels.GithubPullRequest{})
- repoIdGen := didgen.NewDomainIdGenerator(&githubModels.GithubRepo{})
+ prIdGen := didgen.NewDomainIdGenerator(&models.GithubPullRequest{})
+ repoIdGen := didgen.NewDomainIdGenerator(&models.GithubRepo{})
+ userIdGen := didgen.NewDomainIdGenerator(&models.GithubUser{})
converter, err := helper.NewDataConverter(helper.DataConverterArgs{
- InputRowType: reflect.TypeOf(githubModels.GithubPullRequest{}),
+ InputRowType: reflect.TypeOf(models.GithubPullRequest{}),
Input: cursor,
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
@@ -44,7 +45,7 @@
Table: RAW_PULL_REQUEST_TABLE,
},
Convert: func(inputRow interface{}) ([]interface{}, error) {
- pr := inputRow.(*githubModels.GithubPullRequest)
+ pr := inputRow.(*models.GithubPullRequest)
domainPr := &code.PullRequest{
DomainEntity: domainlayer.DomainEntity{
Id: prIdGen.Generate(pr.GithubId),
@@ -53,7 +54,7 @@
Status: pr.State,
Title: pr.Title,
Url: pr.Url,
- AuthorId: pr.AuthorId,
+ AuthorId: userIdGen.Generate(pr.AuthorId),
AuthorName: pr.AuthorName,
Description: pr.Body,
CreatedDate: pr.GithubCreatedAt,
diff --git a/plugins/github/tasks/pull_request_issue_enricher.go b/plugins/github/tasks/pull_request_issue_enricher.go
index c14b413..d8a0038 100644
--- a/plugins/github/tasks/pull_request_issue_enricher.go
+++ b/plugins/github/tasks/pull_request_issue_enricher.go
@@ -87,10 +87,10 @@
continue
}
githubPullRequstIssue := &githubModels.GithubPullRequestIssue{
- PullRequestId: githubPullRequst.GithubId,
- IssueId: issue.GithubId,
- PullNumber: githubPullRequst.Number,
- IssueNumber: issue.Number,
+ PullRequestId: githubPullRequst.GithubId,
+ IssueId: issue.GithubId,
+ PullRequestNumber: githubPullRequst.Number,
+ IssueNumber: issue.Number,
}
results = append(results, githubPullRequstIssue)
}
diff --git a/plugins/github/tasks/pull_request_issues_convertor.go b/plugins/github/tasks/pull_request_issues_convertor.go
index 8b20b98..df20f67 100644
--- a/plugins/github/tasks/pull_request_issues_convertor.go
+++ b/plugins/github/tasks/pull_request_issues_convertor.go
@@ -48,10 +48,10 @@
Convert: func(inputRow interface{}) ([]interface{}, error) {
githubPrIssue := inputRow.(*githubModels.GithubPullRequestIssue)
pullRequestIssue := &crossdomain.PullRequestIssue{
- PullRequestId: prIdGen.Generate(githubPrIssue.PullRequestId),
- IssueId: issueIdGen.Generate(githubPrIssue.IssueId),
- IssueNumber: githubPrIssue.IssueNumber,
- PullNumber: githubPrIssue.PullNumber,
+ PullRequestId: prIdGen.Generate(githubPrIssue.PullRequestId),
+ IssueId: issueIdGen.Generate(githubPrIssue.IssueId),
+ IssueNumber: githubPrIssue.IssueNumber,
+ PullRequestNumber: githubPrIssue.PullRequestNumber,
}
return []interface{}{
pullRequestIssue,
diff --git a/plugins/gitlab/models/gitlab_merge_request.go b/plugins/gitlab/models/gitlab_merge_request.go
index bf9eec3..5356747 100644
--- a/plugins/gitlab/models/gitlab_merge_request.go
+++ b/plugins/gitlab/models/gitlab_merge_request.go
@@ -7,21 +7,27 @@
)
type GitlabMergeRequest struct {
- GitlabId int `gorm:"primaryKey"`
- Iid int `gorm:"index"`
- ProjectId int `gorm:"index"`
+ GitlabId int `gorm:"primaryKey"`
+ Iid int `gorm:"index"`
+ ProjectId int `gorm:"index"`
+ SourceProjectId int
+ TargetProjectId int
State string `gorm:"type:varchar(255)"`
Title string
WebUrl string `gorm:"type:varchar(255)"`
UserNotesCount int
WorkInProgress bool
SourceBranch string `gorm:"type:varchar(255)"`
+ TargetBranch string `gorm:"type:varchar(255)"`
+ MergeCommitSha string `gorm:"type:varchar(255)"`
MergedAt *time.Time
GitlabCreatedAt time.Time
ClosedAt *time.Time
MergedByUsername string `gorm:"type:varchar(255)"`
Description string
- AuthorUsername string `gorm:"type:varchar(255)"`
+ AuthorUsername string `gorm:"type:varchar(255)"`
+ AuthorUserId int
+ Component string `gorm:"type:varchar(255)"`
FirstCommentTime *time.Time `gorm:"comment:Time when the first comment occurred"`
ReviewRounds int `gorm:"comment:How many rounds of review this MR went through"`
common.NoPKModel
diff --git a/plugins/gitlab/models/gitlab_user.go b/plugins/gitlab/models/gitlab_user.go
index 7b13216..fa1e5ce 100644
--- a/plugins/gitlab/models/gitlab_user.go
+++ b/plugins/gitlab/models/gitlab_user.go
@@ -8,7 +8,6 @@
common.NoPKModel
}
-func (GitlabUser) TableName() string{
+func (GitlabUser) TableName() string {
return "_tool_gitlab_users"
}
-
diff --git a/plugins/gitlab/models/migrationscripts/archived/merge_request.go b/plugins/gitlab/models/migrationscripts/archived/merge_request.go
index 95ca4c0..b8ed239 100644
--- a/plugins/gitlab/models/migrationscripts/archived/merge_request.go
+++ b/plugins/gitlab/models/migrationscripts/archived/merge_request.go
@@ -7,23 +7,29 @@
)
type GitlabMergeRequest struct {
- GitlabId int `gorm:"primaryKey"`
- Iid int `gorm:"index"`
- ProjectId int `gorm:"index"`
+ GitlabId int `gorm:"primaryKey"`
+ Iid int `gorm:"index"`
+ ProjectId int `gorm:"index"`
+ SourceProjectId int
+ TargetProjectId int
State string `gorm:"type:varchar(255)"`
Title string
WebUrl string `gorm:"type:varchar(255)"`
UserNotesCount int
WorkInProgress bool
SourceBranch string `gorm:"type:varchar(255)"`
+ TargetBranch string `gorm:"type:varchar(255)"`
+ MergeCommitSha string `gorm:"type:varchar(255)"`
MergedAt *time.Time
GitlabCreatedAt time.Time
ClosedAt *time.Time
MergedByUsername string `gorm:"type:varchar(255)"`
Description string
- AuthorUsername string `gorm:"type:varchar(255)"`
+ AuthorUsername string `gorm:"type:varchar(255)"`
+ AuthorUserId int
+ Component string `gorm:"type:varchar(255)"`
FirstCommentTime *time.Time `gorm:"comment:Time when the first comment occurred"`
- ReviewRounds int `gorm:"comment:How many rounds of review this MR went through"`
+ ReviewRounds int
archived.NoPKModel
}
diff --git a/plugins/gitlab/tasks/gitlab_api_merge_request_converter.go b/plugins/gitlab/tasks/merge_request_converter.go
similarity index 72%
rename from plugins/gitlab/tasks/gitlab_api_merge_request_converter.go
rename to plugins/gitlab/tasks/merge_request_converter.go
index 40303a7..04759e1 100644
--- a/plugins/gitlab/tasks/gitlab_api_merge_request_converter.go
+++ b/plugins/gitlab/tasks/merge_request_converter.go
@@ -24,7 +24,6 @@
domainMrIdGenerator := didgen.NewDomainIdGenerator(&models.GitlabMergeRequest{})
domainRepoIdGenerator := didgen.NewDomainIdGenerator(&models.GitlabProject{})
-
//Find all piplines associated with the current projectid
cursor, err := db.Model(&models.GitlabMergeRequest{}).Where("project_id=?", data.Options.ProjectId).Rows()
if err != nil {
@@ -44,13 +43,21 @@
DomainEntity: domainlayer.DomainEntity{
Id: domainMrIdGenerator.Generate(gitlabMr.GitlabId),
},
- BaseRepoId: domainRepoIdGenerator.Generate(gitlabMr.ProjectId),
- Status: gitlabMr.State,
- Title: gitlabMr.Title,
- Url: gitlabMr.WebUrl,
- CreatedDate: gitlabMr.GitlabCreatedAt,
- MergedDate: gitlabMr.MergedAt,
- ClosedDate: gitlabMr.ClosedAt,
+ BaseRepoId: domainRepoIdGenerator.Generate(gitlabMr.SourceProjectId),
+ HeadRepoId: domainRepoIdGenerator.Generate(gitlabMr.TargetProjectId),
+ Status: gitlabMr.State,
+ Number: gitlabMr.Iid,
+ Title: gitlabMr.Title,
+ Description: gitlabMr.Description,
+ Url: gitlabMr.WebUrl,
+ AuthorName: gitlabMr.AuthorUsername,
+ CreatedDate: gitlabMr.GitlabCreatedAt,
+ MergedDate: gitlabMr.MergedAt,
+ ClosedDate: gitlabMr.ClosedAt,
+ MergeCommitSha: gitlabMr.MergeCommitSha,
+ HeadRef: gitlabMr.SourceBranch,
+ BaseRef: gitlabMr.TargetBranch,
+ Component: gitlabMr.Component,
}
return []interface{}{
diff --git a/plugins/gitlab/tasks/api_merge_request_extractor.go b/plugins/gitlab/tasks/merge_request_extractor.go
similarity index 85%
rename from plugins/gitlab/tasks/api_merge_request_extractor.go
rename to plugins/gitlab/tasks/merge_request_extractor.go
index 42e1a1f..ea0abd1 100644
--- a/plugins/gitlab/tasks/api_merge_request_extractor.go
+++ b/plugins/gitlab/tasks/merge_request_extractor.go
@@ -12,6 +12,8 @@
GitlabId int `json:"id"`
Iid int
ProjectId int `json:"project_id"`
+ SourceProjectId int `json:"source_project_id"`
+ TargetProjectId int `json:"target_project_id"`
State string
Title string
Description string
@@ -19,13 +21,16 @@
UserNotesCount int `json:"user_notes_count"`
WorkInProgress bool `json:"work_in_progress"`
SourceBranch string `json:"source_branch"`
+ TargetBranch string `json:"target_branch"`
GitlabCreatedAt core.Iso8601Time `json:"created_at"`
MergedAt *core.Iso8601Time `json:"merged_at"`
ClosedAt *core.Iso8601Time `json:"closed_at"`
+ MergeCommitSha string `json:"merge_commit_sha"`
MergedBy struct {
Username string `json:"username"`
} `json:"merged_by"`
Author struct {
+ Id int `json:"id"`
Username string `json:"username"`
}
Reviewers []Reviewer
@@ -61,7 +66,7 @@
return nil, err
}
- gitlabMergeRequest, err := convertMergeRequest(mr, data.Options.ProjectId)
+ gitlabMergeRequest, err := convertMergeRequest(mr)
if err != nil {
return nil, err
}
@@ -94,11 +99,13 @@
return extractor.Execute()
}
-func convertMergeRequest(mr *MergeRequestRes, projectId int) (*models.GitlabMergeRequest, error) {
+func convertMergeRequest(mr *MergeRequestRes) (*models.GitlabMergeRequest, error) {
gitlabMergeRequest := &models.GitlabMergeRequest{
GitlabId: mr.GitlabId,
Iid: mr.Iid,
ProjectId: mr.ProjectId,
+ SourceProjectId: mr.SourceProjectId,
+ TargetProjectId: mr.TargetProjectId,
State: mr.State,
Title: mr.Title,
Description: mr.Description,
@@ -106,11 +113,14 @@
UserNotesCount: mr.UserNotesCount,
WorkInProgress: mr.WorkInProgress,
SourceBranch: mr.SourceBranch,
+ TargetBranch: mr.TargetBranch,
+ MergeCommitSha: mr.MergeCommitSha,
MergedAt: core.Iso8601TimeToTime(mr.MergedAt),
GitlabCreatedAt: mr.GitlabCreatedAt.ToTime(),
ClosedAt: core.Iso8601TimeToTime(mr.ClosedAt),
MergedByUsername: mr.MergedBy.Username,
AuthorUsername: mr.Author.Username,
+ AuthorUserId: mr.Author.Id,
}
return gitlabMergeRequest, nil
}
