GithubPlugin: Pass config in taskCtx instead of db models (#2192)
* Pass config in taskCtx instead of db models
* prepare migrate script init_schema for the coming PR which append connectionId
* use inject instead of use config.GetConfig in migration script
Co-authored-by: linyh <yanghui@meri.co>
diff --git a/.env.example b/.env.example
index 0ceca13..80dc9e3 100644
--- a/.env.example
+++ b/.env.example
@@ -80,8 +80,6 @@
########################
# GitHub configuration has been migrated into DB #
-# FIXME It's very special because no defined in config, so it will be deleted in next PR
-GITHUB_PR_BODY_CLOSE_PATTERN='(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\s]*.*(((and )?(#|https:\/\/github.com\/%s\/%s\/issues\/)\d+[ ]*)+)'
# FIXME this config use in refdiff
GITHUB_PR_TITLE_PATTERN='.*\(#(\d+)\)'
diff --git a/plugins/github/README.md b/plugins/github/README.md
index 0e8660f..847233c 100644
--- a/plugins/github/README.md
+++ b/plugins/github/README.md
@@ -65,8 +65,16 @@
Click **Save Settings** to update additional settings.
### Regular Expression Configuration
-Define regex pattern in .env
-- GITHUB_PR_BODY_CLOSE_PATTERN: Define key word to associate issue in pr body, please check the example in .env.example
+Define regex pattern in request options
+- prType: Define key word to associate issue in pr body
+- prComponent: Define key word to associate issue in pr body
+- prBodyClosePattern: Define key word to associate issue in pr body
+- issueSeverity: Define key word to associate issue in pr body
+- issuePriority: Define key word to associate issue in pr body
+- issueComponent: Define key word to associate issue in pr body
+- issueTypeBug: Define key word to associate issue in pr body
+- issueTypeIncident: Define key word to associate issue in pr body
+- issueTypeRequirement: Define key word to associate issue in pr body
## Sample Request
In order to collect data, you have to compose a JSON looks like following one, and send it by selecting `Advanced Mode` on `Create Pipeline Run` page:
@@ -79,6 +87,7 @@
"options": {
"repo": "lake",
"owner": "merico-dev"
+ // add more config such as prType if necessary.
}
}
]
diff --git a/plugins/github/github.go b/plugins/github/github.go
index 93ddf23..d189912 100644
--- a/plugins/github/github.go
+++ b/plugins/github/github.go
@@ -18,148 +18,49 @@
package main // must be main for plugin entry point
import (
- "fmt"
- "github.com/apache/incubator-devlake/migration"
- "github.com/apache/incubator-devlake/plugins/core"
- "github.com/apache/incubator-devlake/plugins/github/api"
- "github.com/apache/incubator-devlake/plugins/github/models"
- "github.com/apache/incubator-devlake/plugins/github/models/migrationscripts"
- "github.com/apache/incubator-devlake/plugins/github/tasks"
+ "github.com/apache/incubator-devlake/plugins/github/impl"
"github.com/apache/incubator-devlake/runner"
- "github.com/mitchellh/mapstructure"
"github.com/spf13/cobra"
- "github.com/spf13/viper"
- "gorm.io/gorm"
)
-var _ core.PluginMeta = (*Github)(nil)
-var _ core.PluginInit = (*Github)(nil)
-var _ core.PluginTask = (*Github)(nil)
-var _ core.PluginApi = (*Github)(nil)
-var _ core.Migratable = (*Github)(nil)
-
-type Github struct{}
-
-func (plugin Github) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) error {
- api.Init(config, logger, db)
- return nil
-}
-
-func (plugin Github) Description() string {
- return "To collect and enrich data from GitHub"
-}
-
-func (plugin Github) SubTaskMetas() []core.SubTaskMeta {
- return []core.SubTaskMeta{
- tasks.CollectApiRepoMeta,
- tasks.ExtractApiRepoMeta,
- tasks.CollectApiIssuesMeta,
- tasks.ExtractApiIssuesMeta,
- tasks.CollectApiPullRequestsMeta,
- tasks.ExtractApiPullRequestsMeta,
- tasks.CollectApiCommentsMeta,
- tasks.ExtractApiCommentsMeta,
- tasks.CollectApiEventsMeta,
- tasks.ExtractApiEventsMeta,
- tasks.CollectApiPullRequestCommitsMeta,
- tasks.ExtractApiPullRequestCommitsMeta,
- tasks.CollectApiPullRequestReviewsMeta,
- tasks.ExtractApiPullRequestReviewsMeta,
- tasks.CollectApiCommitsMeta,
- tasks.ExtractApiCommitsMeta,
- tasks.CollectApiCommitStatsMeta,
- tasks.ExtractApiCommitStatsMeta,
- tasks.EnrichPullRequestIssuesMeta,
- tasks.ConvertRepoMeta,
- tasks.ConvertIssuesMeta,
- tasks.ConvertCommitsMeta,
- tasks.ConvertIssueLabelsMeta,
- tasks.ConvertPullRequestCommitsMeta,
- tasks.ConvertPullRequestsMeta,
- tasks.ConvertPullRequestLabelsMeta,
- tasks.ConvertPullRequestIssuesMeta,
- tasks.ConvertUsersMeta,
- tasks.ConvertIssueCommentsMeta,
- tasks.ConvertPullRequestCommentsMeta,
- }
-}
-
-func (plugin Github) PrepareTaskData(taskCtx core.TaskContext, options map[string]interface{}) (interface{}, error) {
- var op tasks.GithubOptions
- err := mapstructure.Decode(options, &op)
- if err != nil {
- return nil, err
- }
- if op.Owner == "" {
- return nil, fmt.Errorf("owner is required for GitHub execution")
- }
- if op.Repo == "" {
- return nil, fmt.Errorf("repo is required for GitHub execution")
- }
-
- // find the only github now
- // FIXME to query by connection id when multi connection support
- connection := &models.GithubConnection{}
- err = taskCtx.GetDb().First(connection, "name = ?", `GitHub`).Error
- if err != nil {
- return err, nil
- }
-
- apiClient, err := tasks.CreateApiClient(taskCtx, connection)
- if err != nil {
- return nil, err
- }
-
- return &tasks.GithubTaskData{
- Options: &op,
- ApiClient: apiClient,
- }, nil
-}
-
-func (plugin Github) RootPkgPath() string {
- return "github.com/apache/incubator-devlake/plugins/github"
-}
-
-func (plugin Github) MigrationScripts() []migration.Script {
- return []migration.Script{
- new(migrationscripts.InitSchemas), new(migrationscripts.UpdateSchemas20220509),
- new(migrationscripts.UpdateSchemas20220524), new(migrationscripts.UpdateSchemas20220608),
- }
-}
-
-func (plugin Github) ApiResources() map[string]map[string]core.ApiResourceHandler {
- return map[string]map[string]core.ApiResourceHandler{
- "test": {
- "POST": api.TestConnection,
- },
- "connections": {
- "POST": api.PostConnections,
- "GET": api.ListConnections,
- },
- "connections/:connectionId": {
- "GET": api.GetConnection,
- "PATCH": api.PatchConnection,
- "DELETE": api.DeleteConnection,
- },
- }
-}
-
// Export a variable named PluginEntry for Framework to search and load
-var PluginEntry Github //nolint
+var PluginEntry impl.Github //nolint
// standalone mode for debugging
func main() {
- githubCmd := &cobra.Command{Use: "github"}
- owner := githubCmd.Flags().StringP("owner", "o", "", "github owner")
- repo := githubCmd.Flags().StringP("repo", "r", "", "github repo")
- _ = githubCmd.MarkFlagRequired("owner")
- _ = githubCmd.MarkFlagRequired("repo")
+ cmd := &cobra.Command{Use: "github"}
+ connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "github connection id")
+ owner := cmd.Flags().StringP("owner", "o", "", "github owner")
+ repo := cmd.Flags().StringP("repo", "r", "", "github repo")
+ _ = cmd.MarkFlagRequired("connectionId")
+ _ = cmd.MarkFlagRequired("owner")
+ _ = cmd.MarkFlagRequired("repo")
- githubCmd.Run = func(cmd *cobra.Command, args []string) {
+ prType := cmd.Flags().String("prType", "type/(.*)$", "pr type")
+ prComponent := cmd.Flags().String("prComponent", "component/(.*)$", "pr component")
+ prBodyClosePattern := cmd.Flags().String("prBodyClosePattern", "(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and )?(#|https:\\/\\/github.com\\/%s\\/%s\\/issues\\/)\\d+[ ]*)+)", "pr body close pattern")
+ issueSeverity := cmd.Flags().String("issueSeverity", "severity/(.*)$", "issue severity")
+ issuePriority := cmd.Flags().String("issuePriority", "^(highest|high|medium|low)$", "issue priority")
+ issueComponent := cmd.Flags().String("issueComponent", "component/(.*)$", "issue component")
+ issueTypeBug := cmd.Flags().String("issueTypeBug", "^(bug|failure|error)$", "issue type bug")
+ issueTypeIncident := cmd.Flags().String("issueTypeIncident", "", "issue type incident")
+ issueTypeRequirement := cmd.Flags().String("issueTypeRequirement", "^(feat|feature|proposal|requirement)$", "issue type requirement")
+
+ cmd.Run = func(cmd *cobra.Command, args []string) {
runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
- "owner": *owner,
- "repo": *repo,
+ "connectionId": *connectionId,
+ "owner": *owner,
+ "repo": *repo,
+ "prType": *prType,
+ "prComponent": *prComponent,
+ "prBodyClosePattern": *prBodyClosePattern,
+ "issueSeverity": *issueSeverity,
+ "issuePriority": *issuePriority,
+ "issueComponent": *issueComponent,
+ "issueTypeBug": *issueTypeBug,
+ "issueTypeIncident": *issueTypeIncident,
+ "issueTypeRequirement": *issueTypeRequirement,
})
}
- runner.RunCmd(githubCmd)
+ runner.RunCmd(cmd)
}
diff --git a/plugins/github/impl/impl.go b/plugins/github/impl/impl.go
new file mode 100644
index 0000000..b4f1ff2
--- /dev/null
+++ b/plugins/github/impl/impl.go
@@ -0,0 +1,176 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package impl
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/migration"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/github/api"
+ "github.com/apache/incubator-devlake/plugins/github/models"
+ "github.com/apache/incubator-devlake/plugins/github/models/migrationscripts"
+ "github.com/apache/incubator-devlake/plugins/github/tasks"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/mitchellh/mapstructure"
+ "github.com/spf13/viper"
+ "gorm.io/gorm"
+)
+
+var _ core.PluginMeta = (*Github)(nil)
+var _ core.PluginInit = (*Github)(nil)
+var _ core.PluginTask = (*Github)(nil)
+var _ core.PluginApi = (*Github)(nil)
+var _ core.Migratable = (*Github)(nil)
+
+type Github struct{}
+
+func (plugin Github) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) error {
+ api.Init(config, logger, db)
+ return nil
+}
+
+func (plugin Github) Description() string {
+ return "To collect and enrich data from GitHub"
+}
+
+func (plugin Github) SubTaskMetas() []core.SubTaskMeta {
+ return []core.SubTaskMeta{
+ tasks.CollectApiRepoMeta,
+ tasks.ExtractApiRepoMeta,
+ tasks.CollectApiIssuesMeta,
+ tasks.ExtractApiIssuesMeta,
+ tasks.CollectApiPullRequestsMeta,
+ tasks.ExtractApiPullRequestsMeta,
+ tasks.CollectApiCommentsMeta,
+ tasks.ExtractApiCommentsMeta,
+ tasks.CollectApiEventsMeta,
+ tasks.ExtractApiEventsMeta,
+ tasks.CollectApiPullRequestCommitsMeta,
+ tasks.ExtractApiPullRequestCommitsMeta,
+ tasks.CollectApiPullRequestReviewsMeta,
+ tasks.ExtractApiPullRequestReviewsMeta,
+ tasks.CollectApiCommitsMeta,
+ tasks.ExtractApiCommitsMeta,
+ tasks.CollectApiCommitStatsMeta,
+ tasks.ExtractApiCommitStatsMeta,
+ tasks.EnrichPullRequestIssuesMeta,
+ tasks.ConvertRepoMeta,
+ tasks.ConvertIssuesMeta,
+ tasks.ConvertCommitsMeta,
+ tasks.ConvertIssueLabelsMeta,
+ tasks.ConvertPullRequestCommitsMeta,
+ tasks.ConvertPullRequestsMeta,
+ tasks.ConvertPullRequestLabelsMeta,
+ tasks.ConvertPullRequestIssuesMeta,
+ tasks.ConvertUsersMeta,
+ tasks.ConvertIssueCommentsMeta,
+ tasks.ConvertPullRequestCommentsMeta,
+ }
+}
+
+func (plugin Github) PrepareTaskData(taskCtx core.TaskContext, options map[string]interface{}) (interface{}, error) {
+ var op tasks.GithubOptions
+ err := mapstructure.Decode(options, &op)
+ if err != nil {
+ return nil, err
+ }
+ if op.Owner == "" {
+ return nil, fmt.Errorf("owner is required for GitHub execution")
+ }
+ if op.Repo == "" {
+ return nil, fmt.Errorf("repo is required for GitHub execution")
+ }
+ if op.PrType == "" {
+ op.PrType = "type/(.*)$"
+ }
+ if op.PrComponent == "" {
+ op.PrComponent = "component/(.*)$"
+ }
+ if op.PrBodyClosePattern == "" {
+ op.PrBodyClosePattern = "(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and )?(#|https:\\/\\/github.com\\/%s\\/%s\\/issues\\/)\\d+[ ]*)+)"
+ }
+ if op.IssueSeverity == "" {
+ op.IssueSeverity = "severity/(.*)$"
+ }
+ if op.IssuePriority == "" {
+ op.IssuePriority = "^(highest|high|medium|low)$"
+ }
+ if op.IssueComponent == "" {
+ op.IssueComponent = "component/(.*)$"
+ }
+ if op.IssueTypeBug == "" {
+ op.IssueTypeBug = "^(bug|failure|error)$"
+ }
+ if op.IssueTypeIncident == "" {
+ op.IssueTypeIncident = ""
+ }
+ if op.IssueTypeRequirement == "" {
+ op.IssueTypeRequirement = "^(feat|feature|proposal|requirement)$"
+ }
+
+ // find the needed GitHub now
+ if op.ConnectionId == 0 {
+ return nil, fmt.Errorf("connectionId is invalid")
+ }
+ connectionHelper := helper.NewConnectionHelper(
+ taskCtx,
+ nil,
+ )
+ connection := &models.GithubConnection{}
+ err = connectionHelper.FirstById(connection, op.ConnectionId)
+ if err != nil {
+ return err, nil
+ }
+
+ apiClient, err := tasks.CreateApiClient(taskCtx, connection)
+ if err != nil {
+ return nil, err
+ }
+
+ return &tasks.GithubTaskData{
+ Options: &op,
+ ApiClient: apiClient,
+ }, nil
+}
+
+func (plugin Github) RootPkgPath() string {
+ return "github.com/apache/incubator-devlake/plugins/github"
+}
+
+func (plugin Github) MigrationScripts() []migration.Script {
+ return []migration.Script{
+ new(migrationscripts.InitSchemas),
+ }
+}
+
+func (plugin Github) ApiResources() map[string]map[string]core.ApiResourceHandler {
+ return map[string]map[string]core.ApiResourceHandler{
+ "test": {
+ "POST": api.TestConnection,
+ },
+ "connections": {
+ "POST": api.PostConnections,
+ "GET": api.ListConnections,
+ },
+ "connections/:connectionId": {
+ "GET": api.GetConnection,
+ "PATCH": api.PatchConnection,
+ "DELETE": api.DeleteConnection,
+ },
+ }
+}
diff --git a/plugins/github/models/connection.go b/plugins/github/models/connection.go
index 34303c9..afc7577 100644
--- a/plugins/github/models/connection.go
+++ b/plugins/github/models/connection.go
@@ -33,6 +33,7 @@
type Config struct {
PrType string `mapstructure:"prType" json:"prType"`
PrComponent string `mapstructure:"prComponent" json:"prComponent"`
+ PrBodyClosePattern string `mapstructure:"prBodyClosePattern" json:"prBodyClosePattern"`
IssueSeverity string `mapstructure:"issueSeverity" json:"issueSeverity"`
IssuePriority string `mapstructure:"issuePriority" json:"issuePriority"`
IssueComponent string `mapstructure:"issueComponent" json:"issueComponent"`
diff --git a/plugins/github/models/migrationscripts/archived/commit_stat.go b/plugins/github/models/migrationscripts/archived/commit_stat.go
index 828d4db..b928070 100644
--- a/plugins/github/models/migrationscripts/archived/commit_stat.go
+++ b/plugins/github/models/migrationscripts/archived/commit_stat.go
@@ -24,9 +24,10 @@
)
type GithubCommitStat struct {
- Sha string `gorm:"primaryKey;type:varchar(40)"`
- Additions int `gorm:"comment:Added lines of code"`
- Deletions int `gorm:"comment:Deleted lines of code"`
+ Sha string `gorm:"primaryKey;type:varchar(40)"`
+ Additions int `gorm:"comment:Added lines of code"`
+ Deletions int `gorm:"comment:Deleted lines of code"`
+
CommittedDate time.Time `gorm:"index"`
archived.NoPKModel
}
diff --git a/plugins/github/models/migrationscripts/archived/issue.go b/plugins/github/models/migrationscripts/archived/issue.go
index 17a49fc..ff465b8 100644
--- a/plugins/github/models/migrationscripts/archived/issue.go
+++ b/plugins/github/models/migrationscripts/archived/issue.go
@@ -33,6 +33,8 @@
Priority string `gorm:"type:varchar(255)"`
Type string `gorm:"type:varchar(100)"`
Status string `gorm:"type:varchar(255)"`
+ AuthorId int
+ AuthorName string `gorm:"type:varchar(255)"`
AssigneeId int
AssigneeName string `gorm:"type:varchar(255)"`
LeadTimeMinutes uint
diff --git a/plugins/github/models/migrationscripts/archived/pull_request.go b/plugins/github/models/migrationscripts/archived/pull_request.go
index 43895e7..ef4ff04 100644
--- a/plugins/github/models/migrationscripts/archived/pull_request.go
+++ b/plugins/github/models/migrationscripts/archived/pull_request.go
@@ -50,7 +50,7 @@
HeadCommitSha string `gorm:"type:varchar(255)"`
Url string `gorm:"type:varchar(255)"`
AuthorName string `gorm:"type:varchar(100)"`
- AuthorId string `gorm:"type:varchar(100)"`
+ AuthorId int
archived.NoPKModel
}
diff --git a/plugins/github/models/migrationscripts/archived/repo.go b/plugins/github/models/migrationscripts/archived/repo.go
index 2a514a2..7ecee73 100644
--- a/plugins/github/models/migrationscripts/archived/repo.go
+++ b/plugins/github/models/migrationscripts/archived/repo.go
@@ -24,9 +24,9 @@
)
type GithubRepo struct {
- GithubId int `gorm:"primaryKey"`
- Name string
- HTMLUrl string
+ GithubId int `gorm:"primaryKey"`
+ Name string `gorm:"type:varchar(255)"`
+ HTMLUrl string `gorm:"type:varchar(255)"`
Description string
OwnerId int `json:"ownerId"`
OwnerLogin string `json:"ownerLogin" gorm:"type:varchar(255)"`
diff --git a/plugins/github/models/migrationscripts/init_schema.go b/plugins/github/models/migrationscripts/init_schema.go
deleted file mode 100644
index b924cb2..0000000
--- a/plugins/github/models/migrationscripts/init_schema.go
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package migrationscripts
-
-import (
- "context"
-
- "github.com/apache/incubator-devlake/plugins/github/models/migrationscripts/archived"
- "gorm.io/gorm"
-)
-
-type InitSchemas struct{}
-
-func (*InitSchemas) Up(ctx context.Context, db *gorm.DB) error {
- return db.Migrator().AutoMigrate(
- &archived.GithubRepo{},
- &archived.GithubCommit{},
- &archived.GithubRepoCommit{},
- &archived.GithubPullRequest{},
- &archived.GithubReviewer{},
- &archived.GithubPullRequestComment{},
- &archived.GithubPullRequestCommit{},
- &archived.GithubPullRequestLabel{},
- &archived.GithubIssue{},
- &archived.GithubIssueComment{},
- &archived.GithubIssueEvent{},
- &archived.GithubIssueLabel{},
- &archived.GithubUser{},
- &archived.GithubPullRequestIssue{},
- &archived.GithubCommitStat{})
-}
-
-func (*InitSchemas) Version() uint64 {
- return 20220407201135
-}
-
-func (*InitSchemas) Name() string {
- return "Github init schemas"
-}
diff --git a/plugins/github/models/migrationscripts/init_schema_20220610.go b/plugins/github/models/migrationscripts/init_schema_20220610.go
new file mode 100644
index 0000000..d684f76
--- /dev/null
+++ b/plugins/github/models/migrationscripts/init_schema_20220610.go
@@ -0,0 +1,139 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "context"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+
+ commonArchived "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+ "github.com/apache/incubator-devlake/plugins/github/models/migrationscripts/archived"
+ "gorm.io/gorm"
+)
+
+type GithubConnection20220609 struct {
+ commonArchived.Model
+ Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"`
+ Endpoint string `mapstructure:"endpoint" env:"GITHUB_ENDPOINT" validate:"required"`
+ Proxy string `mapstructure:"proxy" env:"GITHUB_PROXY"`
+ RateLimit int `comment:"api request rate limit per hour"`
+ Token string `mapstructure:"token" env:"GITHUB_AUTH" validate:"required" encrypt:"yes"`
+}
+
+func (GithubConnection20220609) TableName() string {
+ return "_tool_github_connections"
+}
+
+type InitSchemas struct {
+ config core.ConfigGetter
+}
+
+func (u *InitSchemas) SetConfigGetter(config core.ConfigGetter) {
+ u.config = config
+}
+
+func (u *InitSchemas) Up(ctx context.Context, db *gorm.DB) error {
+ if db.Migrator().HasTable(GithubConnection20220609{}) {
+ err := db.Migrator().DropTable(GithubConnection20220609{})
+ if err != nil {
+ return err
+ }
+ }
+ err := db.Migrator().DropTable(
+ &archived.GithubRepo{},
+ &archived.GithubCommit{},
+ &archived.GithubRepoCommit{},
+ &archived.GithubPullRequest{},
+ &archived.GithubReviewer{},
+ &archived.GithubPullRequestComment{},
+ &archived.GithubPullRequestCommit{},
+ &archived.GithubPullRequestLabel{},
+ &archived.GithubIssue{},
+ &archived.GithubIssueComment{},
+ &archived.GithubIssueEvent{},
+ &archived.GithubIssueLabel{},
+ &archived.GithubUser{},
+ &archived.GithubPullRequestIssue{},
+ &archived.GithubCommitStat{},
+ "_raw_github_api_issues",
+ "_raw_github_api_comments",
+ "_raw_github_api_commits",
+ "_raw_github_api_commit_stats",
+ "_raw_github_api_events",
+ "_raw_github_api_issues",
+ "_raw_github_api_pull_requests",
+ "_raw_github_api_pull_request_commits",
+ "_raw_github_api_pull_request_reviews",
+ "_raw_github_api_repositories",
+ )
+
+ // create connection
+ if err != nil {
+ return err
+ }
+ err = db.Migrator().CreateTable(GithubConnection20220609{})
+ if err != nil {
+ return err
+ }
+ connection := &GithubConnection20220609{}
+ connection.Endpoint = u.config.GetString(`GITHUB_ENDPOINT`)
+ connection.Proxy = u.config.GetString(`GITHUB_PROXY`)
+ connection.Token = u.config.GetString(`GITHUB_AUTH`)
+ connection.Name = `GitHub`
+ if err != nil {
+ return err
+ }
+ err = helper.UpdateEncryptFields(connection, func(plaintext string) (string, error) {
+ return core.Encrypt(u.config.GetString(core.EncodeKeyEnvStr), plaintext)
+ })
+ if err != nil {
+ return err
+ }
+ // update from .env and save to db
+ if connection.Endpoint != `` && connection.Token != `` {
+ db.Create(connection)
+ }
+
+ // create other table with connection id
+ return db.Migrator().AutoMigrate(
+ &archived.GithubRepo{},
+ &archived.GithubCommit{},
+ &archived.GithubRepoCommit{},
+ &archived.GithubPullRequest{},
+ &archived.GithubReviewer{},
+ &archived.GithubPullRequestComment{},
+ &archived.GithubPullRequestCommit{},
+ &archived.GithubPullRequestLabel{},
+ &archived.GithubIssue{},
+ &archived.GithubIssueComment{},
+ &archived.GithubIssueEvent{},
+ &archived.GithubIssueLabel{},
+ &archived.GithubUser{},
+ &archived.GithubPullRequestIssue{},
+ &archived.GithubCommitStat{},
+ )
+}
+
+func (*InitSchemas) Version() uint64 {
+ return 20220610000001
+}
+
+func (*InitSchemas) Name() string {
+ return "Github init schemas 20220610"
+}
diff --git a/plugins/github/models/migrationscripts/updateSchemas20220509.go b/plugins/github/models/migrationscripts/updateSchemas20220509.go
deleted file mode 100644
index c4f7603..0000000
--- a/plugins/github/models/migrationscripts/updateSchemas20220509.go
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package migrationscripts
-
-import (
- "context"
- "github.com/apache/incubator-devlake/models/migrationscripts/archived"
- "time"
-
- "gorm.io/gorm"
-)
-
-type GithubPullRequest20220509 struct {
- GithubId int `gorm:"primaryKey"`
- RepoId int `gorm:"index"`
- Number int `gorm:"index"` // This number is used in GET requests to the API associated to reviewers / comments / etc.
- State string `gorm:"type:varchar(255)"`
- Title string `gorm:"type:varchar(255)"`
- GithubCreatedAt time.Time
- GithubUpdatedAt time.Time `gorm:"index"`
- ClosedAt *time.Time
- // In order to get the following fields, we need to collect PRs individually from GitHub
- Additions int
- Deletions int
- Comments int
- Commits int
- ReviewComments int
- Merged bool
- MergedAt *time.Time
- Body string
- Type string `gorm:"type:varchar(255)"`
- Component string `gorm:"type:varchar(255)"`
- MergeCommitSha string `gorm:"type:varchar(40)"`
- HeadRef string `gorm:"type:varchar(255)"`
- BaseRef string `gorm:"type:varchar(255)"`
- BaseCommitSha string `gorm:"type:varchar(255)"`
- HeadCommitSha string `gorm:"type:varchar(255)"`
- Url string `gorm:"type:varchar(255)"`
- AuthorName string `gorm:"type:varchar(100)"`
- AuthorId int
- archived.NoPKModel
-}
-
-func (GithubPullRequest20220509) TableName() string {
- return "_tool_github_pull_requests"
-}
-
-type UpdateSchemas20220509 struct{}
-
-func (*UpdateSchemas20220509) Up(ctx context.Context, db *gorm.DB) error {
- if db.Name() == "postgres" {
- err := db.Exec("ALTER TABLE _tool_github_pull_requests ALTER COLUMN author_id TYPE bigint USING author_id::bigint").Error
- if err != nil {
- return err
- }
- return nil
- }
- err := db.Migrator().AlterColumn(GithubPullRequest20220509{}, "author_id")
- if err != nil {
- return err
- }
- return nil
-}
-
-func (*UpdateSchemas20220509) Version() uint64 {
- return 20220509212344
-}
-
-func (*UpdateSchemas20220509) Name() string {
- return "Alter pr column `author_id` to int"
-}
diff --git a/plugins/github/models/migrationscripts/updateSchemas20220524.go b/plugins/github/models/migrationscripts/updateSchemas20220524.go
deleted file mode 100644
index 2196cab..0000000
--- a/plugins/github/models/migrationscripts/updateSchemas20220524.go
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package migrationscripts
-
-import (
- "context"
- "gorm.io/gorm"
-)
-
-type GithubIssue20220524 struct {
- AuthorId int
- AuthorName string `gorm:"type:varchar(255)"`
-}
-
-func (GithubIssue20220524) TableName() string {
- return "_tool_github_issues"
-}
-
-type UpdateSchemas20220524 struct{}
-
-func (*UpdateSchemas20220524) Up(ctx context.Context, db *gorm.DB) error {
- err := db.Migrator().AddColumn(GithubIssue20220524{}, "author_id")
- if err != nil {
- return err
- }
- err = db.Migrator().AddColumn(GithubIssue20220524{}, "author_name")
- if err != nil {
- return err
- }
- return nil
-}
-
-func (*UpdateSchemas20220524) Version() uint64 {
- return 20220524000002
-}
-
-func (*UpdateSchemas20220524) Name() string {
- return "Add column `author_id`/`author_name` in `GithubIssue`"
-}
diff --git a/plugins/github/models/migrationscripts/updateSchemas20220608.go b/plugins/github/models/migrationscripts/updateSchemas20220608.go
deleted file mode 100644
index 679579b..0000000
--- a/plugins/github/models/migrationscripts/updateSchemas20220608.go
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package migrationscripts
-
-import (
- "context"
- "github.com/apache/incubator-devlake/config"
- "github.com/apache/incubator-devlake/models/migrationscripts/archived"
- "github.com/apache/incubator-devlake/plugins/helper"
- "gorm.io/gorm"
-)
-
-type GithubConnection20220608 struct {
- archived.Model
- Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"`
- Endpoint string `mapstructure:"endpoint" env:"GITHUB_ENDPOINT" validate:"required"`
- Proxy string `mapstructure:"proxy" env:"GITHUB_PROXY"`
- RateLimit int `comment:"api request rate limit per hour"`
- Token string `mapstructure:"token" validate:"required" env:"GITHUB_AUTH"`
-
- Config20220608 `mapstructure:",squash"`
-}
-
-type Config20220608 struct {
- PrType string `mapstructure:"prType" env:"GITHUB_PR_TYPE" json:"prType"`
- PrComponent string `mapstructure:"prComponent" env:"GITHUB_PR_COMPONENT" json:"prComponent"`
- IssueSeverity string `mapstructure:"issueSeverity" env:"GITHUB_ISSUE_SEVERITY" json:"issueSeverity"`
- IssuePriority string `mapstructure:"issuePriority" env:"GITHUB_ISSUE_PRIORITY" json:"issuePriority"`
- IssueComponent string `mapstructure:"issueComponent" env:"GITHUB_ISSUE_COMPONENT" json:"issueComponent"`
- IssueTypeBug string `mapstructure:"issueTypeBug" env:"GITHUB_ISSUE_TYPE_BUG" json:"issueTypeBug"`
- IssueTypeIncident string `mapstructure:"issueTypeIncident" env:"GITHUB_ISSUE_TYPE_INCIDENT" json:"issueTypeIncident"`
- IssueTypeRequirement string `mapstructure:"issueTypeRequirement" env:"GITHUB_ISSUE_TYPE_REQUIREMENT" json:"issueTypeRequirement"`
-}
-
-func (GithubConnection20220608) TableName() string {
- return "_tool_github_connections"
-}
-
-type UpdateSchemas20220608 struct{}
-
-func (*UpdateSchemas20220608) Up(ctx context.Context, db *gorm.DB) error {
- err := db.Migrator().CreateTable(GithubConnection20220608{})
- if err != nil {
- return err
- }
- v := config.GetConfig()
- connection := &GithubConnection20220608{}
- err = helper.EncodeStruct(v, connection, "env")
- connection.Name = `GitHub`
- if err != nil {
- return err
- }
- // update from .env and save to db
- if connection.Endpoint != `` && connection.Token != `` {
- db.Create(connection)
- }
- return nil
-}
-
-func (*UpdateSchemas20220608) Version() uint64 {
- return 20220608000003
-}
-
-func (*UpdateSchemas20220608) Name() string {
- return "Add connection for github"
-}
diff --git a/plugins/github/tasks/comment_collector.go b/plugins/github/tasks/comment_collector.go
index e7c2298..9263a31 100644
--- a/plugins/github/tasks/comment_collector.go
+++ b/plugins/github/tasks/comment_collector.go
@@ -31,7 +31,7 @@
const RAW_COMMENTS_TABLE = "github_api_comments"
-// this struct should be moved to `gitub_api_common.go`
+// this struct should be moved to `github_api_common.go`
func CollectApiComments(taskCtx core.SubTaskContext) error {
db := taskCtx.GetDb()
@@ -72,7 +72,6 @@
since = &latestUpdatedPrComt.GithubUpdatedAt
incremental = true
}
-
}
collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
diff --git a/plugins/github/tasks/pr_issue_enricher.go b/plugins/github/tasks/pr_issue_enricher.go
index 6fa108d..f16beac 100644
--- a/plugins/github/tasks/pr_issue_enricher.go
+++ b/plugins/github/tasks/pr_issue_enricher.go
@@ -40,7 +40,7 @@
repoId := data.Repo.GithubId
var prBodyCloseRegex *regexp.Regexp
- prBodyClosePattern := taskCtx.GetConfig("GITHUB_PR_BODY_CLOSE_PATTERN")
+ prBodyClosePattern := data.Options.PrBodyClosePattern
//the pattern before the issue number, sometimes, the issue number is #1098, sometimes it is https://xxx/#1098
prBodyClosePattern = strings.Replace(prBodyClosePattern, "%s", data.Options.Owner, 1)
prBodyClosePattern = strings.Replace(prBodyClosePattern, "%s", data.Options.Repo, 1)
diff --git a/plugins/github/tasks/task_data.go b/plugins/github/tasks/task_data.go
index f48c77b..eb551dc 100644
--- a/plugins/github/tasks/task_data.go
+++ b/plugins/github/tasks/task_data.go
@@ -25,11 +25,12 @@
)
type GithubOptions struct {
- Tasks []string `json:"tasks,omitempty"`
- Since string
- Owner string
- Repo string
- models.Config
+ ConnectionId uint64 `json:"connectionId"`
+ Tasks []string `json:"tasks,omitempty"`
+ Since string
+ Owner string
+ Repo string
+ models.Config `mapstructure:",squash"`
}
type GithubTaskData struct {
