Merge pull request #1062 from merico-dev/fix1061
Fix Github commits data lost
diff --git a/models/domainlayer/ticket/board.go b/models/domainlayer/ticket/board.go
index 07ddf4b..9399c40 100644
--- a/models/domainlayer/ticket/board.go
+++ b/models/domainlayer/ticket/board.go
@@ -1,11 +1,20 @@
package ticket
import (
+ "time"
+
"github.com/merico-dev/lake/models/domainlayer"
)
type Board struct {
domainlayer.DomainEntity
- Name string
- Url string
+ Name string
+ Description string
+ Url string
+ CreatedDate *time.Time
+}
+
+type BoardSprint struct {
+ BoardId string `gorm:"primaryKey"`
+ SprintId string `gorm:"primaryKey"`
}
diff --git a/models/domainlayer/ticket/issue.go b/models/domainlayer/ticket/issue.go
index 479f971..9fcbf99 100644
--- a/models/domainlayer/ticket/issue.go
+++ b/models/domainlayer/ticket/issue.go
@@ -8,27 +8,23 @@
type Issue struct {
domainlayer.DomainEntity
-
- // collected fields
- Url string
- Key string
- Title string
- Summary string
- EpicKey string
- Type string
- Status string
- StoryPoint uint
- OriginalEstimateMinutes int64 // user input?
- AggregateEstimateMinutes int64 // sum up of all subtasks?
- RemainingEstimateMinutes int64 // could it be negative value?
- CreatorId string
- AssigneeId string
- ResolutionDate *time.Time
- Priority string // not sure how to deal with it yet, copy the name for now
- ParentId string
- SprintId string
- CreatedDate time.Time
- UpdatedDate time.Time
- SpentMinutes int64
- LeadTimeMinutes uint
+ Url string
+ Key string
+ Title string
+ Summary string
+ EpicKey string
+ Type string
+ Status string
+ StoryPoint uint
+ ResolutionDate *time.Time
+ CreatedDate *time.Time
+ UpdatedDate *time.Time
+ LeadTimeMinutes uint
+ ParentIssueId string
+ Priority string
+ OriginalEstimateMinutes int64
+ TimeRemainingMinutes int64
+ CreatorId string
+ AssigneeId string
+ OwnerId string
}
diff --git a/models/domainlayer/ticket/issue_history.go b/models/domainlayer/ticket/issue_history.go
new file mode 100644
index 0000000..2b66e27
--- /dev/null
+++ b/models/domainlayer/ticket/issue_history.go
@@ -0,0 +1,43 @@
+package ticket
+
+import (
+ "time"
+
+ "github.com/merico-dev/lake/models/common"
+)
+
+type IssueStatusHistory struct {
+ common.NoPKModel
+ IssueId string `gorm:"primaryKey"`
+ Status string `gorm:"primaryKey"`
+ StartDate time.Time `gorm:"primaryKey"`
+ EndDate *time.Time
+}
+
+func (IssueStatusHistory) TableName() string {
+ return "issue_status_history"
+}
+
+type IssueAssigneeHistory struct {
+ common.NoPKModel
+ IssueId string `gorm:"primaryKey"`
+ Assignee string `gorm:"primaryKey"`
+ StartDate time.Time `gorm:"primaryKey"`
+ EndDate *time.Time
+}
+
+func (IssueAssigneeHistory) TableName() string {
+ return "issue_assignee_history"
+}
+
+type IssueSprintsHistory struct {
+ common.NoPKModel
+ IssueId string `gorm:"primaryKey"`
+ SprintId string `gorm:"primaryKey"`
+ StartDate time.Time `gorm:"primaryKey"`
+ EndDate *time.Time
+}
+
+func (IssueSprintsHistory) TableName() string {
+ return "issue_sprints_history"
+}
diff --git a/models/domainlayer/ticket/sprint.go b/models/domainlayer/ticket/sprint.go
index 88e985c..db42653 100644
--- a/models/domainlayer/ticket/sprint.go
+++ b/models/domainlayer/ticket/sprint.go
@@ -6,57 +6,29 @@
"github.com/merico-dev/lake/models/domainlayer"
)
+const (
+ BeforeSprint = "BEFORE_SPRINT"
+ DuringSprint = "DURING_SPRINT"
+ AfterSprint = "AFTER_SPRINT"
+)
+
type Sprint struct {
domainlayer.DomainEntity
-
- // collected fields
- BoardId string `gorm:"index"`
- Url string
- State string
- Name string
- StartDate *time.Time
- EndDate *time.Time
- CompleteDate *time.Time
+ Name string
+ Url string
+ Status string
+ Title string
+ StartedDate *time.Time
+ EndedDate *time.Time
+ CompletedDate *time.Time
}
type SprintIssue struct {
- SprintId string `gorm:"primaryKey"`
- IssueId string `gorm:"primaryKey"`
- AddedAt *time.Time
- RemovedAt *time.Time
-}
-
-type SprintIssueBurndown struct {
- SprintId string `gorm:"primaryKey"`
- EndedHour int `gorm:"primaryKey"`
- StartedAt time.Time
- EndedAt time.Time
-
- Added int
- Removed int
- Remaining int
-
- AddedRequirements int
- RemovedRequirements int
- RemainingRequirements int
-
- AddedBugs int
- RemovedBugs int
- RemainingBugs int
-
- AddedIncidents int
- RemovedIncidents int
- RemainingIncidents int
-
- AddedOtherIssues int
- RemovedOtherIssues int
- RemainingOtherIssues int
-
- AddedStoryPoints int
- RemovedStoryPoints int
- RemainingStoryPoints int
-}
-
-func (SprintIssueBurndown) TableName() string {
- return "sprint_issue_burndown"
-}
+ SprintId string `gorm:"primaryKey"`
+ IssueId string `gorm:"primaryKey"`
+ IsRemoved bool
+ AddedDate *time.Time
+ RemovedDate *time.Time
+ AddedStage string
+ ResolvedStage string
+}
\ No newline at end of file
diff --git a/models/domainlayer/ticket/worklog.go b/models/domainlayer/ticket/worklog.go
index 1aab868..31b3a77 100644
--- a/models/domainlayer/ticket/worklog.go
+++ b/models/domainlayer/ticket/worklog.go
@@ -8,12 +8,10 @@
type Worklog struct {
domainlayer.DomainEntity
- IssueId string `gorm:"index"`
- BoardId string `gorm:"index"`
AuthorId string
- UpdateAuthorId string
- TimeSpent string
- TimeSpentSeconds int
- Updated time.Time
- Started time.Time
+ Comment string
+ TimeSpentMinutes int
+ LoggedDate *time.Time
+ StartedDate *time.Time
+ IssueId string `gorm:"index"`
}
diff --git a/models/init.go b/models/init.go
index 543dad6..a6fbb92 100644
--- a/models/init.go
+++ b/models/init.go
@@ -60,10 +60,13 @@
&ticket.Board{},
&ticket.Issue{},
&ticket.BoardIssue{},
+ &ticket.BoardSprint{},
&ticket.Changelog{},
&ticket.Sprint{},
&ticket.SprintIssue{},
- &ticket.SprintIssueBurndown{},
+ &ticket.IssueStatusHistory{},
+ &ticket.IssueSprintsHistory{},
+ &ticket.IssueAssigneeHistory{},
&devops.Job{},
&devops.Build{},
&ticket.Worklog{},
diff --git a/models/test/changelog_model_test.go b/models/test/changelog_model_test.go
index 93cec9a..18bca39 100644
--- a/models/test/changelog_model_test.go
+++ b/models/test/changelog_model_test.go
@@ -9,11 +9,11 @@
)
func TestInsertChangelog(t *testing.T) {
- board, err := factory.CreateBoard()
+ _, err := factory.CreateBoard()
assert.Nil(t, err)
- issue, err := factory.CreateIssue(board.DomainEntity.Id)
+ issue, err := factory.CreateIssue()
assert.Nil(t, err)
- changelog, err := factory.CreateChangelog(issue.DomainEntity.Id)
+ changelog, err := factory.CreateChangelog(issue.Id)
assert.Nil(t, err)
tx := models.Db.Create(&changelog)
assert.Nil(t, tx.Error)
diff --git a/models/test/factory/issue_factory.go b/models/test/factory/issue_factory.go
index d3df9ae..fee08ab 100644
--- a/models/test/factory/issue_factory.go
+++ b/models/test/factory/issue_factory.go
@@ -7,7 +7,8 @@
"github.com/merico-dev/lake/models/domainlayer/ticket"
)
-func CreateIssue(boardId string) (*ticket.Issue, error) {
+func CreateIssue() (*ticket.Issue, error) {
+ now := time.Now()
issue := &ticket.Issue{
DomainEntity: domainlayer.DomainEntity{
Id: RandIntString(),
@@ -21,17 +22,13 @@
Status: "",
StoryPoint: 1,
OriginalEstimateMinutes: 1, // user input?
- AggregateEstimateMinutes: 1, // sum up of all subtasks?
- RemainingEstimateMinutes: 1, // could it be negative value?
CreatorId: "",
AssigneeId: "",
ResolutionDate: nil,
Priority: "", // not sure how to deal with it yet, copy the name for now
- ParentId: "",
- SprintId: "",
- CreatedDate: time.Now(),
- UpdatedDate: time.Now(),
- SpentMinutes: 1,
+ ParentIssueId: "",
+ CreatedDate: &now,
+ UpdatedDate: &now,
LeadTimeMinutes: 1,
}
return issue, nil
diff --git a/models/test/factory/sprint_factory.go b/models/test/factory/sprint_factory.go
index 2a8ec4a..b860034 100644
--- a/models/test/factory/sprint_factory.go
+++ b/models/test/factory/sprint_factory.go
@@ -10,13 +10,12 @@
DomainEntity: domainlayer.DomainEntity{
Id: RandIntString(),
},
- BoardId: boardId, // ref to board
- Url: "",
- State: "",
- Name: "",
- StartDate: nil,
- EndDate: nil,
- CompleteDate: nil,
+ Url: "",
+ Status: "",
+ Name: "",
+ StartedDate: nil,
+ EndedDate: nil,
+ CompletedDate: nil,
}
return sprint, nil
}
diff --git a/models/test/factory/worklog_factory.go b/models/test/factory/worklog_factory.go
index 161a0e3..ca5ea6e 100644
--- a/models/test/factory/worklog_factory.go
+++ b/models/test/factory/worklog_factory.go
@@ -1,8 +1,6 @@
package factory
import (
- "time"
-
"github.com/merico-dev/lake/models/domainlayer"
"github.com/merico-dev/lake/models/domainlayer/ticket"
)
@@ -12,14 +10,8 @@
DomainEntity: domainlayer.DomainEntity{
Id: RandIntString(),
},
- IssueId: issueId, // ref to issue
- BoardId: boardId, // ref to board
- AuthorId: "",
- UpdateAuthorId: "",
- TimeSpent: "",
- TimeSpentSeconds: RandInt(),
- Updated: time.Now(),
- Started: time.Now(),
+ IssueId: issueId, // ref to issue
+ AuthorId: "",
}
return worklog, nil
}
diff --git a/models/test/issue_model_test.go b/models/test/issue_model_test.go
index 2eaf58c..c22ff5f 100644
--- a/models/test/issue_model_test.go
+++ b/models/test/issue_model_test.go
@@ -9,9 +9,7 @@
)
func TestInsertIssue(t *testing.T) {
- board, err := factory.CreateBoard()
- assert.Nil(t, err)
- issue, err := factory.CreateIssue(board.DomainEntity.Id)
+ issue, err := factory.CreateIssue()
assert.Nil(t, err)
tx := models.Db.Create(&issue)
assert.Nil(t, tx.Error)
diff --git a/models/test/worklog_model_test.go b/models/test/worklog_model_test.go
index da8c36d..bc9f848 100644
--- a/models/test/worklog_model_test.go
+++ b/models/test/worklog_model_test.go
@@ -11,9 +11,9 @@
func TestInsertWorklog(t *testing.T) {
board, err := factory.CreateBoard()
assert.Nil(t, err)
- issue, err := factory.CreateIssue(board.DomainEntity.Id)
+ issue, err := factory.CreateIssue()
assert.Nil(t, err)
- worklog, err := factory.CreateWorklog(board.DomainEntity.Id, issue.DomainEntity.Id)
+ worklog, err := factory.CreateWorklog(board.Id, issue.Id)
assert.Nil(t, err)
tx := models.Db.Create(&worklog)
assert.Nil(t, tx.Error)
diff --git a/plugins/github/tasks/github_issue_converter.go b/plugins/github/tasks/github_issue_converter.go
index bbe2bfc..22e9e4d 100644
--- a/plugins/github/tasks/github_issue_converter.go
+++ b/plugins/github/tasks/github_issue_converter.go
@@ -2,9 +2,9 @@
import (
"fmt"
+ "github.com/merico-dev/lake/models/domainlayer"
lakeModels "github.com/merico-dev/lake/models"
- "github.com/merico-dev/lake/models/domainlayer"
"github.com/merico-dev/lake/models/domainlayer/didgen"
"github.com/merico-dev/lake/models/domainlayer/ticket"
githubModels "github.com/merico-dev/lake/plugins/github/models"
@@ -38,9 +38,7 @@
func convertToIssueModel(issue *githubModels.GithubIssue) *ticket.Issue {
domainIssue := &ticket.Issue{
- DomainEntity: domainlayer.DomainEntity{
- Id: didgen.NewDomainIdGenerator(issue).Generate(issue.GithubId),
- },
+ DomainEntity:domainlayer.DomainEntity{Id: didgen.NewDomainIdGenerator(issue).Generate(issue.GithubId)},
Key: fmt.Sprint(issue.GithubId),
Title: issue.Title,
Summary: issue.Body,
@@ -49,8 +47,8 @@
Type: issue.Type,
AssigneeId: issue.Assignee,
LeadTimeMinutes: issue.LeadTimeMinutes,
- CreatedDate: issue.GithubCreatedAt,
- UpdatedDate: issue.GithubUpdatedAt,
+ CreatedDate: &issue.GithubCreatedAt,
+ UpdatedDate: &issue.GithubUpdatedAt,
ResolutionDate: issue.ClosedAt,
}
return domainIssue
diff --git a/plugins/jira/tasks/jira_board_converter.go b/plugins/jira/tasks/jira_board_converter.go
index 867e685..be29825 100644
--- a/plugins/jira/tasks/jira_board_converter.go
+++ b/plugins/jira/tasks/jira_board_converter.go
@@ -18,9 +18,7 @@
}
board := &ticket.Board{
- DomainEntity: domainlayer.DomainEntity{
- Id: didgen.NewDomainIdGenerator(jiraBoard).Generate(jiraBoard.SourceId, boardId),
- },
+ DomainEntity:domainlayer.DomainEntity{Id: didgen.NewDomainIdGenerator(jiraBoard).Generate(jiraBoard.SourceId, boardId)},
Name: jiraBoard.Name,
Url: jiraBoard.Self,
}
diff --git a/plugins/jira/tasks/jira_changelog_converter.go b/plugins/jira/tasks/jira_changelog_converter.go
index 0e49b31..c4813fa 100644
--- a/plugins/jira/tasks/jira_changelog_converter.go
+++ b/plugins/jira/tasks/jira_changelog_converter.go
@@ -65,7 +65,7 @@
logger.Info("convert changelog", fmt.Sprintf("%s .. %s", batch[0].Id, batch[i-1].Id))
return nil
}
- burndownConverter := NewSprintIssueBurndownConverter()
+ sprintIssueConverter := NewSprintIssueConverter()
row := &ChangelogItemResult{}
// iterate all rows
for cursor.Next() {
@@ -93,7 +93,7 @@
changelog.To = row.ToString
changelog.CreatedDate = row.Created
i++
- burndownConverter.FeedIn(sourceId, *row)
+ sprintIssueConverter.FeedIn(sourceId, *row)
}
if i > 0 {
err = saveBatch()
@@ -101,11 +101,7 @@
return err
}
}
- err = burndownConverter.Save()
- if err != nil {
- logger.Error("save error:", err)
- }
- err = burndownConverter.UpdateSprintIssue()
+ err = sprintIssueConverter.UpdateSprintIssue()
if err != nil {
logger.Error("update sprint issue error:", err)
}
diff --git a/plugins/jira/tasks/jira_issue_converter.go b/plugins/jira/tasks/jira_issue_converter.go
index 73d0c41..bdc606c 100644
--- a/plugins/jira/tasks/jira_issue_converter.go
+++ b/plugins/jira/tasks/jira_issue_converter.go
@@ -25,7 +25,6 @@
issueIdGen := didgen.NewDomainIdGenerator(&jiraModels.JiraIssue{})
userIdGen := didgen.NewDomainIdGenerator(&jiraModels.JiraUser{})
- sprintIdGen := didgen.NewDomainIdGenerator(&jiraModels.JiraSprint{})
boardIssue := &ticket.BoardIssue{
BoardId: didgen.NewDomainIdGenerator(&jiraModels.JiraBoard{}).Generate(sourceId, boardId),
@@ -46,32 +45,26 @@
DomainEntity: domainlayer.DomainEntity{
Id: issueIdGen.Generate(jiraIssue.SourceId, jiraIssue.IssueId),
},
- Url: jiraIssue.Self,
- Key: jiraIssue.Key,
- Summary: jiraIssue.Summary,
- EpicKey: jiraIssue.EpicKey,
- Type: jiraIssue.StdType,
- Status: jiraIssue.StdStatus,
- StoryPoint: jiraIssue.StdStoryPoint,
- OriginalEstimateMinutes: jiraIssue.OriginalEstimateMinutes,
- AggregateEstimateMinutes: jiraIssue.AggregateEstimateMinutes,
- RemainingEstimateMinutes: jiraIssue.RemainingEstimateMinutes,
- CreatorId: userIdGen.Generate(sourceId, jiraIssue.CreatorAccountId),
- ResolutionDate: jiraIssue.ResolutionDate,
- Priority: jiraIssue.PriorityName,
- CreatedDate: jiraIssue.Created,
- UpdatedDate: jiraIssue.Updated,
- LeadTimeMinutes: jiraIssue.LeadTimeMinutes,
- SpentMinutes: jiraIssue.SpentMinutes,
+ Url: jiraIssue.Self,
+ Key: jiraIssue.Key,
+ Summary: jiraIssue.Summary,
+ EpicKey: jiraIssue.EpicKey,
+ Type: jiraIssue.StdType,
+ Status: jiraIssue.StdStatus,
+ StoryPoint: jiraIssue.StdStoryPoint,
+ OriginalEstimateMinutes: jiraIssue.OriginalEstimateMinutes,
+ CreatorId: userIdGen.Generate(sourceId, jiraIssue.CreatorAccountId),
+ ResolutionDate: jiraIssue.ResolutionDate,
+ Priority: jiraIssue.PriorityName,
+ CreatedDate: &jiraIssue.Created,
+ UpdatedDate: &jiraIssue.Updated,
+ LeadTimeMinutes: jiraIssue.LeadTimeMinutes,
}
if jiraIssue.AssigneeAccountId != "" {
issue.AssigneeId = userIdGen.Generate(sourceId, jiraIssue.AssigneeAccountId)
}
if jiraIssue.ParentId != 0 {
- issue.ParentId = issueIdGen.Generate(sourceId, jiraIssue.ParentId)
- }
- if jiraIssue.SprintId != 0 {
- issue.SprintId = sprintIdGen.Generate(sourceId, jiraIssue.SprintId)
+ issue.ParentIssueId = issueIdGen.Generate(sourceId, jiraIssue.ParentId)
}
err = lakeModels.Db.Clauses(clause.OnConflict{UpdateAll: true}).Create(issue).Error
diff --git a/plugins/jira/tasks/jira_sprint_converter.go b/plugins/jira/tasks/jira_sprint_converter.go
index e826ef5..be398ce 100644
--- a/plugins/jira/tasks/jira_sprint_converter.go
+++ b/plugins/jira/tasks/jira_sprint_converter.go
@@ -22,7 +22,7 @@
}
defer cursor.Close()
- boardIdGen := didgen.NewDomainIdGenerator(&jiraModels.JiraBoard{}).Generate(sourceId, boardId)
+ domainBoardId := didgen.NewDomainIdGenerator(&jiraModels.JiraBoard{}).Generate(sourceId, boardId)
sprintIdGen := didgen.NewDomainIdGenerator(&jiraModels.JiraSprint{})
issueIdGen := didgen.NewDomainIdGenerator(&jiraModels.JiraIssue{})
// iterate all rows
@@ -33,16 +33,13 @@
return err
}
sprint := &ticket.Sprint{
- DomainEntity: domainlayer.DomainEntity{
- Id: sprintIdGen.Generate(jiraSprint.SourceId, jiraSprint.SprintId),
- },
- BoardId: boardIdGen,
- Url: jiraSprint.Self,
- State: jiraSprint.State,
- Name: jiraSprint.Name,
- StartDate: jiraSprint.StartDate,
- EndDate: jiraSprint.EndDate,
- CompleteDate: jiraSprint.CompleteDate,
+ DomainEntity:domainlayer.DomainEntity{Id: sprintIdGen.Generate(jiraSprint.SourceId, jiraSprint.SprintId)},
+ Url: jiraSprint.Self,
+ Status: jiraSprint.State,
+ Name: jiraSprint.Name,
+ StartedDate: jiraSprint.StartDate,
+ EndedDate: jiraSprint.EndDate,
+ CompletedDate: jiraSprint.CompleteDate,
}
err = lakeModels.Db.Clauses(clause.OnConflict{UpdateAll: true}).Create(sprint).Error
if err != nil {
@@ -65,6 +62,14 @@
if err != nil {
return err
}
+ boardSprint := &ticket.BoardSprint{
+ BoardId: domainBoardId,
+ SprintId: sprint.Id,
+ }
+ err = lakeModels.Db.Clauses(clause.OnConflict{DoNothing: true}).Create(boardSprint).Error
+ if err != nil {
+ return err
+ }
}
return nil
}
diff --git a/plugins/jira/tasks/jira_sprint_issue_burndown_converter.go b/plugins/jira/tasks/jira_sprint_issue_burndown_converter.go
deleted file mode 100644
index e65bc77..0000000
--- a/plugins/jira/tasks/jira_sprint_issue_burndown_converter.go
+++ /dev/null
@@ -1,349 +0,0 @@
-package tasks
-
-import (
- "fmt"
- "github.com/merico-dev/lake/logger"
- "gorm.io/gorm"
- "gorm.io/gorm/clause"
- "math"
- "strconv"
- "strings"
- "time"
-
- lakeModels "github.com/merico-dev/lake/models"
- "github.com/merico-dev/lake/models/domainlayer/didgen"
- "github.com/merico-dev/lake/models/domainlayer/ticket"
- "github.com/merico-dev/lake/plugins/jira/models"
-)
-
-// issue types
-const (
- Bug = "Bug"
- Story = "Story"
- Incident = "Incident"
- Task = "Task"
-)
-
-const (
- BashSize = 100
-)
-
-var (
- UTCLocation, _ = time.LoadLocation("UTC")
-)
-
-type SprintIssueBurndownConverter struct {
- cache map[string]map[int]*ticket.SprintIssueBurndown
- sprintIdGen *didgen.DomainIdGenerator
- issueIdGen *didgen.DomainIdGenerator
- sprints map[string]*models.JiraSprint
- sprintIssue map[string]*ticket.SprintIssue
-}
-
-func NewSprintIssueBurndownConverter() *SprintIssueBurndownConverter {
- return &SprintIssueBurndownConverter{
- cache: make(map[string]map[int]*ticket.SprintIssueBurndown),
- sprintIdGen: didgen.NewDomainIdGenerator(&models.JiraSprint{}),
- issueIdGen: didgen.NewDomainIdGenerator(&models.JiraIssue{}),
- sprints: make(map[string]*models.JiraSprint),
- sprintIssue: make(map[string]*ticket.SprintIssue),
- }
-}
-
-func (c *SprintIssueBurndownConverter) FeedIn(sourceId uint64, cl ChangelogItemResult) {
- if cl.Field != "Sprint" {
- return
- }
- from, to, err := c.parseFromTo(cl.From, cl.To)
- if err != nil {
- return
- }
- for sprintId := range from {
- err = c.handleFrom(sourceId, sprintId, cl)
- if err != nil {
- logger.Error("handle from error:", err)
- return
- }
- }
- for sprintId := range to {
- err = c.handleTo(sourceId, sprintId, cl)
- if err != nil {
- logger.Error("handle to error:", err)
- return
- }
- }
-}
-
-func (c *SprintIssueBurndownConverter) UpdateSprintIssue() error {
- var err error
- var flag bool
- var list []*ticket.SprintIssue
- for _, fresh := range c.sprintIssue {
- var old ticket.SprintIssue
- err = lakeModels.Db.First(&old, "sprint_id = ? AND issue_id = ?", fresh.SprintId, fresh.IssueId).Error
- if err != nil && err != gorm.ErrRecordNotFound {
- logger.Error("UpdateSprintIssue error:", err)
- return err
- }
-
- if old.AddedAt == nil && fresh.AddedAt != nil || old.RemovedAt == nil && fresh.RemovedAt != nil {
- flag = true
- }
- if old.AddedAt != nil && fresh.AddedAt != nil && old.AddedAt.Before(*fresh.AddedAt) {
- fresh.AddedAt = old.AddedAt
- flag = true
- }
- if old.RemovedAt != nil && fresh.RemovedAt != nil && old.RemovedAt.After(*fresh.RemovedAt) {
- fresh.RemovedAt = old.RemovedAt
- flag = true
- }
- if flag {
- list = append(list, fresh)
- }
- }
- return lakeModels.Db.Clauses(clause.OnConflict{
- UpdateAll: true,
- }).CreateInBatches(list, BatchSize).Error
-}
-func (c *SprintIssueBurndownConverter) Save() error {
- var err error
- for sprintId, sprint := range c.cache {
- err = lakeModels.Db.Clauses(clause.OnConflict{
- UpdateAll: true,
- }).CreateInBatches(c.fill(sprintId, sprint), BatchSize).Error
- if err != nil {
- logger.Error("save sprint issue burndwon error:", err)
- return err
- }
- }
- return nil
-}
-
-func (c *SprintIssueBurndownConverter) fill(sprintId string, m map[int]*ticket.SprintIssueBurndown) []*ticket.SprintIssueBurndown {
- var result []*ticket.SprintIssueBurndown
- var max, min int
- min = math.MaxInt32
- for k := range m {
- if k > max {
- max = k
- }
- if k < min {
- min = k
- }
- }
- for dateHour := min; dateHour <= max; dateHour = c.nextDateHour(dateHour) {
- if item, ok := m[dateHour]; ok {
- result = append(result, item)
- } else {
- result = append(result, c.newSprintIssueBurndown(sprintId, dateHour))
- }
- }
-
- // fill remaining
- var remain, remainBugs, remainRequirements, remainIncidents, remainStoryPoints int
- for _, item := range result {
- remain += item.Added
- remain -= item.Removed
- remainBugs += item.AddedBugs
- remainBugs -= item.RemovedBugs
- remainRequirements += item.AddedRequirements
- remainRequirements -= item.RemovedRequirements
- remainIncidents += item.AddedIncidents
- remainIncidents -= item.RemovedIncidents
- remainStoryPoints += item.AddedStoryPoints
- remainStoryPoints -= item.RemovedStoryPoints
- item.Remaining = remain
- item.RemainingBugs = remainBugs
- item.RemainingRequirements = remainRequirements
- item.RemainingIncidents = remainIncidents
- item.RemainingStoryPoints = remainStoryPoints
- item.RemainingOtherIssues = remain - remainBugs - remainIncidents - remainRequirements - remainStoryPoints
- }
- for p := len(result) - 1; p > -1; p-- {
- for i := 1; i < 24 && p-i > -1; i++ {
- result[p].Added += result[p-i].Added
- result[p].Removed += result[p-i].Removed
- result[p].AddedBugs += result[p-i].AddedBugs
- result[p].RemovedBugs += result[p-i].RemovedBugs
- result[p].AddedRequirements += result[p-i].AddedRequirements
- result[p].RemovedRequirements += result[p-i].RemovedRequirements
- result[p].AddedIncidents += result[p-i].AddedIncidents
- result[p].RemovedIncidents += result[p-i].RemovedIncidents
- result[p].AddedStoryPoints += result[p-i].AddedStoryPoints
- result[p].RemovedStoryPoints += result[p-i].RemovedStoryPoints
- result[p].AddedOtherIssues += result[p-i].AddedOtherIssues
- result[p].RemovedOtherIssues += result[p-i].RemovedOtherIssues
- }
- }
- return result
-}
-func (c *SprintIssueBurndownConverter) getJiraIssue(sourceId, issueId uint64) (*models.JiraIssue, error) {
- var issue models.JiraIssue
- err := lakeModels.Db.First(&issue, "issue_id = ? AND source_id = ?", issueId, sourceId).Error
- if err != nil {
- logger.Error("getJiraIssue error:", err)
- return nil, err
- }
- return &issue, err
-}
-
-
-
-func (c *SprintIssueBurndownConverter) parseFromTo(from, to string) (map[uint64]struct{}, map[uint64]struct{}, error) {
- fromInts := make(map[uint64]struct{})
- toInts := make(map[uint64]struct{})
- var n uint64
- var err error
- for _, item := range strings.Split(from, ",") {
- s := strings.TrimSpace(item)
- if s == ""{
- continue
- }
- n, err = strconv.ParseUint(s, 10, 64)
- if err != nil {
- return nil, nil, err
- }
- fromInts[n] = struct{}{}
- }
- for _, item := range strings.Split(to, ",") {
- s := strings.TrimSpace(item)
- if s == ""{
- continue
- }
- n, err = strconv.ParseUint(s, 10, 64)
- if err != nil {
- return nil, nil, err
- }
- toInts[n] = struct{}{}
- }
- inter := make(map[uint64]struct{})
- for k := range fromInts {
- if _, ok := toInts[k]; ok {
- inter[k] = struct{}{}
- delete(toInts, k)
- }
- }
- for k := range inter {
- delete(fromInts, k)
- }
- return fromInts, toInts, nil
-}
-
-func (c *SprintIssueBurndownConverter) getDateHour(t time.Time) int {
- t = t.UTC().Add(time.Hour)
- y, m, d := t.Date()
- return y*1000000 + int(m)*10000 + d*100 + t.Hour()
-}
-
-func (c *SprintIssueBurndownConverter) dateHourEndPoints(dateHour int) (time.Time, time.Time) {
- y := dateHour / 1000000
- m := (dateHour / 10000) % 100
- d := (dateHour / 100) % 100
- h := dateHour % 100
- end := time.Date(y, time.Month(m), d, h, 0, 0, 0, UTCLocation)
- return end.Add(-24 * time.Hour), end
-}
-
-func (c *SprintIssueBurndownConverter) nextDateHour(dateHour int) int {
- y := dateHour / 1000000
- m := (dateHour / 10000) % 100
- d := (dateHour / 100) % 100
- h := dateHour % 100
- t := time.Date(y, time.Month(m), d, h, 0, 0, 0, UTCLocation).Add(time.Hour).UTC()
- y1, m1, d1 := t.Date()
- return y1*1000000 + int(m1)*10000 + d1*100 + t.Hour()
-}
-
-func (c *SprintIssueBurndownConverter) handleFrom(sourceId, sprintId uint64, cl ChangelogItemResult) error {
- sprint := c.sprintIdGen.Generate(sourceId, sprintId)
- key := fmt.Sprintf("%d:%d:%d", sourceId, sprintId, cl.IssueId)
- if item, ok := c.sprintIssue[key]; ok {
- if item != nil && (item.RemovedAt == nil || item.RemovedAt != nil && item.RemovedAt.Before(cl.Created)) {
- item.RemovedAt = &cl.Created
- }
- } else {
- c.sprintIssue[key] = &ticket.SprintIssue{
- SprintId: sprint,
- IssueId: c.issueIdGen.Generate(sourceId, cl.IssueId),
- AddedAt: nil,
- RemovedAt: &cl.Created,
- }
- }
- dateHour := c.getDateHour(cl.Created)
- if _, ok := c.cache[sprint]; !ok {
- c.cache[sprint] = make(map[int]*ticket.SprintIssueBurndown)
- }
- if c.cache[sprint][dateHour] == nil {
- c.cache[sprint][dateHour] = c.newSprintIssueBurndown(sprint, dateHour)
- }
- c.cache[sprint][dateHour].Removed++
- jiraIssue, err := c.getJiraIssue(sourceId, cl.IssueId)
- if err != nil {
- return err
- }
- switch jiraIssue.StdType {
- case Bug:
- c.cache[sprint][dateHour].RemovedBugs++
- case Incident:
- c.cache[sprint][dateHour].RemovedIncidents++
- case Task:
- c.cache[sprint][dateHour].RemovedRequirements++
- case Story:
- c.cache[sprint][dateHour].RemovedStoryPoints++
- default:
- c.cache[sprint][dateHour].RemovedOtherIssues++
- }
- return nil
-}
-
-func (c *SprintIssueBurndownConverter) handleTo(sourceId, sprintId uint64, cl ChangelogItemResult) error {
- sprint := c.sprintIdGen.Generate(sourceId, sprintId)
- key := fmt.Sprintf("%d:%d:%d", sourceId, sprintId, cl.IssueId)
- if item, ok := c.sprintIssue[key]; ok {
- if item != nil && (item.AddedAt == nil || item.AddedAt != nil && item.AddedAt.After(cl.Created)) {
- item.AddedAt = &cl.Created
- }
- } else {
- c.sprintIssue[key] = &ticket.SprintIssue{
- SprintId: sprint,
- IssueId: c.issueIdGen.Generate(sourceId, cl.IssueId),
- AddedAt: &cl.Created,
- RemovedAt: nil,
- }
- }
- dateHour := c.getDateHour(cl.Created)
- if _, ok := c.cache[sprint]; !ok {
- c.cache[sprint] = make(map[int]*ticket.SprintIssueBurndown)
- }
- if c.cache[sprint][dateHour] == nil {
- c.cache[sprint][dateHour] = c.newSprintIssueBurndown(sprint, dateHour)
- }
- c.cache[sprint][dateHour].Added++
- jiraIssue, err := c.getJiraIssue(sourceId, cl.IssueId)
- if err != nil {
- return err
- }
- switch jiraIssue.StdType {
- case Bug:
- c.cache[sprint][dateHour].AddedBugs++
- case Incident:
- c.cache[sprint][dateHour].AddedIncidents++
- case Task:
- c.cache[sprint][dateHour].AddedRequirements++
- case Story:
- c.cache[sprint][dateHour].AddedStoryPoints++
- default:
- c.cache[sprint][dateHour].AddedOtherIssues++
- }
- return nil
-}
-
-func (c *SprintIssueBurndownConverter) newSprintIssueBurndown(sprintId string, dateHour int) *ticket.SprintIssueBurndown {
- startedAt, endedAt := c.dateHourEndPoints(dateHour)
- return &ticket.SprintIssueBurndown{
- SprintId: sprintId,
- StartedAt: startedAt,
- EndedAt: endedAt,
- EndedHour: dateHour,
- }
-}
diff --git a/plugins/jira/tasks/jira_sprint_issues_converter.go b/plugins/jira/tasks/jira_sprint_issues_converter.go
new file mode 100644
index 0000000..f65cabd
--- /dev/null
+++ b/plugins/jira/tasks/jira_sprint_issues_converter.go
@@ -0,0 +1,276 @@
+package tasks
+
+import (
+ "fmt"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/merico-dev/lake/logger"
+ lakeModels "github.com/merico-dev/lake/models"
+ "github.com/merico-dev/lake/models/domainlayer/didgen"
+ "github.com/merico-dev/lake/models/domainlayer/ticket"
+ "github.com/merico-dev/lake/plugins/jira/models"
+ "gorm.io/gorm"
+ "gorm.io/gorm/clause"
+)
+
+type SprintIssuesConverter struct {
+ sprintIdGen *didgen.DomainIdGenerator
+ issueIdGen *didgen.DomainIdGenerator
+ sprints map[string]*models.JiraSprint
+ sprintIssue map[string]*ticket.SprintIssue
+ status map[string]*ticket.IssueStatusHistory
+ assignee map[string]*ticket.IssueAssigneeHistory
+ sprintsHistory map[string]*ticket.IssueSprintsHistory
+}
+
+func NewSprintIssueConverter() *SprintIssuesConverter {
+ return &SprintIssuesConverter{
+ sprintIdGen: didgen.NewDomainIdGenerator(&models.JiraSprint{}),
+ issueIdGen: didgen.NewDomainIdGenerator(&models.JiraIssue{}),
+ sprints: make(map[string]*models.JiraSprint),
+ sprintIssue: make(map[string]*ticket.SprintIssue),
+ status: make(map[string]*ticket.IssueStatusHistory),
+ assignee: make(map[string]*ticket.IssueAssigneeHistory),
+ sprintsHistory: make(map[string]*ticket.IssueSprintsHistory),
+ }
+}
+
+func (c *SprintIssuesConverter) FeedIn(sourceId uint64, cl ChangelogItemResult) {
+ if cl.Field == "status" {
+ err := c.handleStatus(sourceId, cl)
+ if err != nil {
+ return
+ }
+ }
+ if cl.Field == "assignee" {
+ err := c.handleAssignee(sourceId, cl)
+ if err != nil {
+ return
+ }
+ }
+ if cl.Field != "Sprint" {
+ return
+ }
+ from, to, err := c.parseFromTo(cl.From, cl.To)
+ if err != nil {
+ return
+ }
+ for sprintId := range from {
+ err = c.handleFrom(sourceId, sprintId, cl)
+ if err != nil {
+ logger.Error("handle from error:", err)
+ return
+ }
+ }
+ for sprintId := range to {
+ err = c.handleTo(sourceId, sprintId, cl)
+ if err != nil {
+ logger.Error("handle to error:", err)
+ return
+ }
+ }
+}
+
+func (c *SprintIssuesConverter) UpdateSprintIssue() error {
+ var err error
+ var flag bool
+ var list []*ticket.SprintIssue
+ for _, fresh := range c.sprintIssue {
+ var old ticket.SprintIssue
+ err = lakeModels.Db.First(&old, "sprint_id = ? AND issue_id = ?", fresh.SprintId, fresh.IssueId).Error
+ if err != nil && err != gorm.ErrRecordNotFound {
+ logger.Error("UpdateSprintIssue error:", err)
+ return err
+ }
+
+ if old.AddedDate == nil && fresh.AddedDate != nil || old.RemovedDate == nil && fresh.RemovedDate != nil {
+ flag = true
+ }
+ if old.AddedDate != nil && fresh.AddedDate != nil && old.AddedDate.Before(*fresh.AddedDate) {
+ fresh.AddedDate = old.AddedDate
+ flag = true
+ }
+ if old.RemovedDate != nil && fresh.RemovedDate != nil && old.RemovedDate.After(*fresh.RemovedDate) {
+ fresh.RemovedDate = old.RemovedDate
+ flag = true
+ }
+ if fresh.AddedDate != nil && fresh.RemovedDate != nil {
+ fresh.IsRemoved = fresh.AddedDate.Before(*fresh.RemovedDate)
+ if fresh.IsRemoved != old.IsRemoved {
+ flag = true
+ }
+ }
+ if flag {
+ list = append(list, fresh)
+ }
+ }
+ return lakeModels.Db.Clauses(clause.OnConflict{
+ UpdateAll: true,
+ }).CreateInBatches(list, BatchSize).Error
+}
+
+func (c *SprintIssuesConverter) parseFromTo(from, to string) (map[uint64]struct{}, map[uint64]struct{}, error) {
+ fromInts := make(map[uint64]struct{})
+ toInts := make(map[uint64]struct{})
+ var n uint64
+ var err error
+ for _, item := range strings.Split(from, ",") {
+ s := strings.TrimSpace(item)
+ if s == "" {
+ continue
+ }
+ n, err = strconv.ParseUint(s, 10, 64)
+ if err != nil {
+ return nil, nil, err
+ }
+ fromInts[n] = struct{}{}
+ }
+ for _, item := range strings.Split(to, ",") {
+ s := strings.TrimSpace(item)
+ if s == "" {
+ continue
+ }
+ n, err = strconv.ParseUint(s, 10, 64)
+ if err != nil {
+ return nil, nil, err
+ }
+ toInts[n] = struct{}{}
+ }
+ inter := make(map[uint64]struct{})
+ for k := range fromInts {
+ if _, ok := toInts[k]; ok {
+ inter[k] = struct{}{}
+ delete(toInts, k)
+ }
+ }
+ for k := range inter {
+ delete(fromInts, k)
+ }
+ return fromInts, toInts, nil
+}
+
+func (c *SprintIssuesConverter) handleFrom(sourceId, sprintId uint64, cl ChangelogItemResult) error {
+ domainSprintId := c.sprintIdGen.Generate(sourceId, sprintId)
+ key := fmt.Sprintf("%d:%d:%d", sourceId, sprintId, cl.IssueId)
+ if item, ok := c.sprintIssue[key]; ok {
+ if item != nil && (item.RemovedDate == nil || item.RemovedDate != nil && item.RemovedDate.Before(cl.Created)) {
+ item.RemovedDate = &cl.Created
+ }
+ } else {
+ c.sprintIssue[key] = &ticket.SprintIssue{
+ SprintId: domainSprintId,
+ IssueId: c.issueIdGen.Generate(sourceId, cl.IssueId),
+ AddedDate: nil,
+ RemovedDate: &cl.Created,
+ }
+ }
+ k := fmt.Sprintf("%d:%d", sprintId, cl.IssueId)
+ if item := c.sprintsHistory[k]; item != nil {
+ item.EndDate = &cl.Created
+ err := lakeModels.Db.Create(item).Error
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (c *SprintIssuesConverter) handleTo(sourceId, sprintId uint64, cl ChangelogItemResult) error {
+ domainSprintId := c.sprintIdGen.Generate(sourceId, sprintId)
+ key := fmt.Sprintf("%d:%d:%d", sourceId, sprintId, cl.IssueId)
+ if item, ok := c.sprintIssue[key]; ok {
+ if item != nil && (item.AddedDate == nil || item.AddedDate != nil && item.AddedDate.After(cl.Created)) {
+ item.AddedDate = &cl.Created
+ item.AddedStage, _ = c.getStage(cl.Created, sourceId, sprintId)
+ }
+ } else {
+ c.sprintIssue[key] = &ticket.SprintIssue{
+ SprintId: domainSprintId,
+ IssueId: c.issueIdGen.Generate(sourceId, cl.IssueId),
+ AddedDate: &cl.Created,
+ RemovedDate: nil,
+ }
+ c.sprintIssue[key].AddedStage, _ = c.getStage(cl.Created, sourceId, sprintId)
+ }
+ k := fmt.Sprintf("%d:%d", sprintId, cl.IssueId)
+ c.sprintsHistory[k] = &ticket.IssueSprintsHistory{
+ IssueId: c.issueIdGen.Generate(sourceId, cl.IssueId),
+ SprintId: domainSprintId,
+ StartDate: cl.Created,
+ EndDate: nil,
+ }
+ return nil
+}
+
+func (c *SprintIssuesConverter) getSprint(sourceId, sprintId uint64) (*models.JiraSprint, error) {
+ id := c.sprintIdGen.Generate(sourceId, sprintId)
+ if value, ok := c.sprints[id]; ok {
+ return value, nil
+ }
+ var sprint models.JiraSprint
+ err := lakeModels.Db.First(&sprint, "source_id = ? AND sprint_id = ?", sourceId, sprintId).Error
+ if err != nil {
+ c.sprints[id] = &sprint
+ }
+ return &sprint, err
+}
+
+func (c *SprintIssuesConverter) getStage(t time.Time, sourceId, sprintId uint64) (string, error) {
+ sprint, err := c.getSprint(sourceId, sprintId)
+ if err != nil {
+ return "", err
+ }
+ if sprint.StartDate != nil {
+ if sprint.StartDate.After(t) {
+ return ticket.BeforeSprint, nil
+ }
+ if sprint.StartDate.Equal(t) || (sprint.CompleteDate != nil && sprint.CompleteDate.Equal(t)) {
+ return ticket.DuringSprint, nil
+ }
+ if sprint.CompleteDate != nil && sprint.StartDate.Before(t) && sprint.CompleteDate.After(t) {
+ return ticket.DuringSprint, nil
+ }
+ }
+ if sprint.CompleteDate != nil && sprint.CompleteDate.Before(t) {
+ return ticket.AfterSprint, nil
+ }
+ return "", nil
+}
+
+func (c *SprintIssuesConverter) handleStatus(sourceId uint64, cl ChangelogItemResult) error {
+ issueId := c.issueIdGen.Generate(sourceId, cl.IssueId)
+ if statusHistory := c.status[issueId]; statusHistory != nil {
+ statusHistory.EndDate = &cl.Created
+ err := lakeModels.Db.Create(statusHistory).Error
+ if err != nil {
+ return err
+ }
+ }
+ c.status[issueId] = &ticket.IssueStatusHistory{
+ IssueId: issueId,
+ Status: cl.ToString,
+ StartDate: cl.Created,
+ EndDate: nil,
+ }
+ return nil
+}
+
+func (c *SprintIssuesConverter) handleAssignee(sourceId uint64, cl ChangelogItemResult) error {
+ issueId := c.issueIdGen.Generate(sourceId, cl.IssueId)
+ if assigneeHistory := c.assignee[issueId]; assigneeHistory != nil {
+ assigneeHistory.EndDate = &cl.Created
+ err := lakeModels.Db.Create(assigneeHistory).Error
+ if err != nil {
+ return err
+ }
+ }
+ c.assignee[issueId] = &ticket.IssueAssigneeHistory{
+ IssueId: issueId,
+ Assignee: cl.To,
+ StartDate: cl.Created,
+ EndDate: nil,
+ }
+ return nil
+}
diff --git a/plugins/jira/tasks/jira_user_converter.go b/plugins/jira/tasks/jira_user_converter.go
index bd019dd..f463d58 100644
--- a/plugins/jira/tasks/jira_user_converter.go
+++ b/plugins/jira/tasks/jira_user_converter.go
@@ -21,7 +21,7 @@
userIdGen := didgen.NewDomainIdGenerator(&jiraModels.JiraUser{})
for _, jiraUser := range jiraUserRows {
- user := &user.User{
+ u := &user.User{
DomainEntity: domainlayer.DomainEntity{
Id: userIdGen.Generate(jiraUser.SourceId, jiraUser.AccountId),
},
@@ -31,7 +31,7 @@
Timezone: jiraUser.Timezone,
}
- err = lakeModels.Db.Clauses(clause.OnConflict{UpdateAll: true}).Create(user).Error
+ err = lakeModels.Db.Clauses(clause.OnConflict{UpdateAll: true}).Create(u).Error
if err != nil {
return err
}
diff --git a/plugins/jira/tasks/jira_worklog_converter.go b/plugins/jira/tasks/jira_worklog_converter.go
index ac88141..8280b2b 100644
--- a/plugins/jira/tasks/jira_worklog_converter.go
+++ b/plugins/jira/tasks/jira_worklog_converter.go
@@ -24,7 +24,6 @@
}
defer cursor.Close()
- boardIdGen := didgen.NewDomainIdGenerator(&jiraModels.JiraBoard{}).Generate(sourceId, boardId)
worklogIdGen := didgen.NewDomainIdGenerator(&jiraModels.JiraWorklog{})
userIdGen := didgen.NewDomainIdGenerator(&jiraModels.JiraUser{})
issueIdGen := didgen.NewDomainIdGenerator(&jiraModels.JiraIssue{})
@@ -35,22 +34,13 @@
return err
}
worklog := &ticket.Worklog{
- DomainEntity: domainlayer.DomainEntity{
- Id: worklogIdGen.Generate(jiraWorklog.SourceId, jiraWorklog.IssueId, jiraWorklog.WorklogId),
- },
+ DomainEntity:domainlayer.DomainEntity{Id: worklogIdGen.Generate(jiraWorklog.SourceId, jiraWorklog.IssueId, jiraWorklog.WorklogId)},
IssueId: issueIdGen.Generate(jiraWorklog.SourceId, jiraWorklog.IssueId),
- BoardId: boardIdGen,
- TimeSpent: jiraWorklog.TimeSpent,
- TimeSpentSeconds: jiraWorklog.TimeSpentSeconds,
- Updated: jiraWorklog.Updated,
- Started: jiraWorklog.Started,
+ TimeSpentMinutes: jiraWorklog.TimeSpentSeconds / 60,
}
if jiraWorklog.AuthorId != "" {
worklog.AuthorId = userIdGen.Generate(sourceId, jiraWorklog.AuthorId)
}
- if jiraWorklog.UpdateAuthorId != "" {
- worklog.UpdateAuthorId = userIdGen.Generate(sourceId, jiraWorklog.UpdateAuthorId)
- }
err = lakeModels.Db.Clauses(clause.OnConflict{UpdateAll: true}).Create(worklog).Error
if err != nil {
