fix: add flag for data convert to dora (#7143)
* fix: add flag for data convert to dora
* fix: some minor optimizations
* fix: add subtask_name filter
diff --git a/backend/core/models/domainlayer/devops/cicd_deployment.go b/backend/core/models/domainlayer/devops/cicd_deployment.go
index c395e32..b66e84f 100644
--- a/backend/core/models/domainlayer/devops/cicd_deployment.go
+++ b/backend/core/models/domainlayer/devops/cicd_deployment.go
@@ -33,6 +33,7 @@
TaskDatesInfo
DurationSec *float64
QueuedDurationSec *float64
+ SubtaskName string `gorm:"type:varchar(255)"`
}
func (CICDDeployment) TableName() string {
diff --git a/backend/core/models/domainlayer/devops/cicd_deployment_commit.go b/backend/core/models/domainlayer/devops/cicd_deployment_commit.go
index 4170268..2afbb86 100644
--- a/backend/core/models/domainlayer/devops/cicd_deployment_commit.go
+++ b/backend/core/models/domainlayer/devops/cicd_deployment_commit.go
@@ -40,6 +40,7 @@
RepoId string `gorm:"type:varchar(255)"`
RepoUrl string `gorm:"index;not null"`
PrevSuccessDeploymentCommitId string `gorm:"type:varchar(255)"`
+ SubtaskName string `gorm:"type:varchar(255)"`
}
func (cicdDeploymentCommit CicdDeploymentCommit) TableName() string {
diff --git a/backend/core/models/migrationscripts/20240308_add_subtaskname_to_deployment.go b/backend/core/models/migrationscripts/20240308_add_subtaskname_to_deployment.go
new file mode 100644
index 0000000..7bac0c3
--- /dev/null
+++ b/backend/core/models/migrationscripts/20240308_add_subtaskname_to_deployment.go
@@ -0,0 +1,63 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*addSubtabknameToDeployment)(nil)
+
+type cicdDeployment20240308 struct {
+ SubtaskName string `gorm:"type:varchar(255)"`
+}
+
+func (cicdDeployment20240308) TableName() string {
+ return "cicd_deployments"
+}
+
+type cicdDeploymentCommits20240308 struct {
+ SubtaskName string `gorm:"type:varchar(255)"`
+}
+
+func (cicdDeploymentCommits20240308) TableName() string {
+ return "cicd_deployment_commits"
+}
+
+type addSubtabknameToDeployment struct{}
+
+func (*addSubtabknameToDeployment) Up(basicRes context.BasicRes) errors.Error {
+ db := basicRes.GetDal()
+ if err := db.AutoMigrate(&cicdDeployment20240308{}); err != nil {
+ return err
+ }
+ if err := db.AutoMigrate(&cicdDeploymentCommits20240308{}); err != nil {
+ return err
+ }
+ return nil
+}
+
+func (*addSubtabknameToDeployment) Version() uint64 {
+ return 20240308142101
+}
+
+func (*addSubtabknameToDeployment) Name() string {
+ return "add subtaskname to deployment"
+}
diff --git a/backend/core/models/migrationscripts/register.go b/backend/core/models/migrationscripts/register.go
index 667f864..7732c01 100644
--- a/backend/core/models/migrationscripts/register.go
+++ b/backend/core/models/migrationscripts/register.go
@@ -106,5 +106,6 @@
new(modfiyFieldsSort),
new(modifyIssueLeadTimeMinutesToUint),
new(addUrgencyToIssues),
+ new(addSubtabknameToDeployment),
}
}
diff --git a/backend/plugins/dora/tasks/change_lead_time_calculator.go b/backend/plugins/dora/tasks/change_lead_time_calculator.go
index c76b3d5..8e29f16 100644
--- a/backend/plugins/dora/tasks/change_lead_time_calculator.go
+++ b/backend/plugins/dora/tasks/change_lead_time_calculator.go
@@ -46,11 +46,11 @@
db := taskCtx.GetDal()
logger := taskCtx.GetLogger()
data := taskCtx.GetData().(*DoraTaskData)
- // // Clear previous results from the project
- // err := db.Exec("DELETE FROM project_pr_metrics WHERE project_name = ? ", data.Options.ProjectName)
- // if err != nil {
- // return errors.Default.Wrap(err, "error deleting previous project_pr_metrics")
- // }
+ // Clear previous results from the project
+ err := db.Exec("DELETE FROM project_pr_metrics WHERE project_name = ? ", data.Options.ProjectName)
+ if err != nil {
+ return errors.Default.Wrap(err, "error deleting previous project_pr_metrics")
+ }
// Get pull requests by repo project_name
var clauses = []dal.Clause{
diff --git a/backend/plugins/dora/tasks/deployment_commits_generator.go b/backend/plugins/dora/tasks/deployment_commits_generator.go
index e8a64e2..e298469 100644
--- a/backend/plugins/dora/tasks/deployment_commits_generator.go
+++ b/backend/plugins/dora/tasks/deployment_commits_generator.go
@@ -30,6 +30,8 @@
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
+const DORAGenerateDeploymentCommits = "dora.generateDeploymentCommits"
+
var DeploymentCommitsGeneratorMeta = plugin.SubTaskMeta{
Name: "generateDeploymentCommits",
EntryPoint: GenerateDeploymentCommits,
@@ -103,32 +105,32 @@
}
if data.Options.ScopeId != nil {
clauses = append(clauses, dal.Where(`p.cicd_scope_id = ?`, data.Options.ScopeId))
- // // Clear previous results from the project
- // deleteSql := `DELETE FROM cicd_deployment_commits WHERE cicd_scope_id = ? ;`
- // err := db.Exec(deleteSql, data.Options.ScopeId)
- // if err != nil {
- // return errors.Default.Wrap(err, "error deleting previous cicd_deployment_commits")
- // }
+ // Clear previous results from the project
+ deleteSql := `DELETE FROM cicd_deployment_commits WHERE cicd_scope_id = ? and subtask_name = ?;`
+ err := db.Exec(deleteSql, data.Options.ScopeId, DORAGenerateDeploymentCommits)
+ if err != nil {
+ return errors.Default.Wrap(err, "error deleting previous cicd_deployment_commits")
+ }
} else {
clauses = append(clauses,
dal.Join("LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = p.cicd_scope_id)"),
dal.Where(`pm.project_name = ?`, data.Options.ProjectName),
)
- // // Clear previous results from the project
- // deleteSql := `DELETE FROM cicd_deployment_commits
- // WHERE cicd_scope_id IN (
- // SELECT cicd_scope_id
- // FROM (
- // SELECT cdc.cicd_scope_id
- // FROM cicd_deployment_commits cdc
- // LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = cdc.cicd_scope_id)
- // WHERE pm.project_name = ?
- // ) AS subquery
- // );`
- // err := db.Exec(deleteSql, data.Options.ProjectName)
- // if err != nil {
- // return errors.Default.Wrap(err, "error deleting previous cicd_deployment_commits")
- // }
+ // Clear previous results from the project
+ deleteSql := `DELETE FROM cicd_deployment_commits
+ WHERE cicd_scope_id IN (
+ SELECT cicd_scope_id
+ FROM (
+ SELECT cdc.cicd_scope_id
+ FROM cicd_deployment_commits cdc
+ LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = cdc.cicd_scope_id)
+ WHERE pm.project_name = ?
+ ) AS subquery
+ ) AND subtask_name = ?;`
+ err := db.Exec(deleteSql, data.Options.ProjectName, DORAGenerateDeploymentCommits)
+ if err != nil {
+ return errors.Default.Wrap(err, "error deleting previous cicd_deployment_commits")
+ }
}
cursor, err := db.Cursor(clauses...)
if err != nil {
@@ -198,6 +200,7 @@
domainDeployCommit.Environment = devops.TESTING
}
}
+ domainDeployCommit.SubtaskName = DORAGenerateDeploymentCommits
return []interface{}{domainDeployCommit}, nil
},
})
diff --git a/backend/plugins/dora/tasks/deployment_generator.go b/backend/plugins/dora/tasks/deployment_generator.go
index cca452d..08102fb 100644
--- a/backend/plugins/dora/tasks/deployment_generator.go
+++ b/backend/plugins/dora/tasks/deployment_generator.go
@@ -28,6 +28,8 @@
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
+const DORAGenerateDeployment = "dora.generateDeployments"
+
var DeploymentGeneratorMeta = plugin.SubTaskMeta{
Name: "generateDeployments",
EntryPoint: GenerateDeployment,
@@ -78,32 +80,32 @@
clauses = append(clauses,
dal.Where("p.cicd_scope_id = ?", data.Options.ScopeId),
)
- // // Clear previous results from the cicd_scope_id
- // deleteSql := `DELETE FROM cicd_deployments WHERE cicd_scope_id = ?;`
- // err := db.Exec(deleteSql, data.Options.ScopeId)
- // if err != nil {
- // return errors.Default.Wrap(err, "error deleting previous deployments")
- // }
+ // Clear previous results from the cicd_scope_id
+ deleteSql := `DELETE FROM cicd_deployments WHERE cicd_scope_id = ? and subtask_name = ?;`
+ err := db.Exec(deleteSql, data.Options.ScopeId, DORAGenerateDeployment)
+ if err != nil {
+ return errors.Default.Wrap(err, "error deleting previous deployments")
+ }
} else {
clauses = append(clauses,
dal.Join("LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = p.cicd_scope_id)"),
dal.Where("pm.project_name = ?", data.Options.ProjectName),
)
- // // Clear previous results from the project
- // deleteSql := `DELETE FROM cicd_deployments
- // WHERE cicd_scope_id IN (
- // SELECT cicd_scope_id
- // FROM (
- // SELECT cd.cicd_scope_id
- // FROM cicd_deployments cd
- // LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = cd.cicd_scope_id)
- // WHERE pm.project_name = ?
- // ) AS subquery
- // );`
- // err := db.Exec(deleteSql, data.Options.ProjectName)
- // if err != nil {
- // return errors.Default.Wrap(err, "error deleting previous deployments")
- // }
+ // Clear previous results from the project
+ deleteSql := `DELETE FROM cicd_deployments
+ WHERE cicd_scope_id IN (
+ SELECT cicd_scope_id
+ FROM (
+ SELECT cd.cicd_scope_id
+ FROM cicd_deployments cd
+ LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = cd.cicd_scope_id)
+ WHERE pm.project_name = ?
+ ) AS subquery
+ ) AND subtask_name = ?;`
+ err := db.Exec(deleteSql, data.Options.ProjectName, DORAGenerateDeployment)
+ if err != nil {
+ return errors.Default.Wrap(err, "error deleting previous deployments")
+ }
}
cursor, err := db.Cursor(clauses...)
@@ -153,6 +155,7 @@
domainDeployment.Environment = devops.TESTING
}
}
+ domainDeployment.SubtaskName = DORAGenerateDeployment
return []interface{}{domainDeployment}, nil
},
})
diff --git a/backend/plugins/dora/tasks/incident_deploy_connector.go b/backend/plugins/dora/tasks/incident_deploy_connector.go
index 2ec9f04c..6d9a136 100644
--- a/backend/plugins/dora/tasks/incident_deploy_connector.go
+++ b/backend/plugins/dora/tasks/incident_deploy_connector.go
@@ -47,11 +47,11 @@
func ConnectIncidentToDeployment(taskCtx plugin.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*DoraTaskData)
- // // Clear previous results from the project
- // err := db.Exec("DELETE FROM project_issue_metrics WHERE project_name = ?", data.Options.ProjectName)
- // if err != nil {
- // return errors.Default.Wrap(err, "error deleting previous project_issue_metrics")
- // }
+ // Clear previous results from the project
+ err := db.Exec("DELETE FROM project_issue_metrics WHERE project_name = ?", data.Options.ProjectName)
+ if err != nil {
+ return errors.Default.Wrap(err, "error deleting previous project_issue_metrics")
+ }
// select all issues belongs to the board
clauses := []dal.Clause{
dal.From(`issues i`),
