Merge pull request #1305 from merico-dev/jira-server-support
fix: handle status code 404
diff --git a/models/domainlayer/ticket/sprint.go b/models/domainlayer/ticket/sprint.go
index 444be31..020276e 100644
--- a/models/domainlayer/ticket/sprint.go
+++ b/models/domainlayer/ticket/sprint.go
@@ -21,6 +21,7 @@
StartedDate *time.Time
EndedDate *time.Time
CompletedDate *time.Time
+ OriginBoardID string
}
type SprintIssue struct {
diff --git a/plugins/core/apiclient.go b/plugins/core/apiclient.go
index ed544d9..4df9942 100644
--- a/plugins/core/apiclient.go
+++ b/plugins/core/apiclient.go
@@ -187,7 +187,7 @@
logger.Print(fmt.Sprintf("[api-client][retry %v] %v %v", retry, method, *uri))
res, err = apiClient.client.Do(req)
if err != nil {
- logger.Error("[api-client] error:%v", err)
+ logger.Error("[api-client] error:", err)
if retry < apiClient.maxRetry-1 {
retry += 1
continue
diff --git a/plugins/jira/tasks/jira_api_client.go b/plugins/jira/tasks/jira_api_client.go
index c890a12..f59b47d 100644
--- a/plugins/jira/tasks/jira_api_client.go
+++ b/plugins/jira/tasks/jira_api_client.go
@@ -25,7 +25,7 @@
map[string]string{
"Authorization": fmt.Sprintf("Basic %v", auth),
},
- 10*time.Second,
+ 20*time.Second,
3,
scheduler,
)
diff --git a/plugins/jira/tasks/jira_remotelink_collector.go b/plugins/jira/tasks/jira_remotelink_collector.go
index f641f1c..4bba7d3 100644
--- a/plugins/jira/tasks/jira_remotelink_collector.go
+++ b/plugins/jira/tasks/jira_remotelink_collector.go
@@ -16,7 +16,7 @@
"github.com/merico-dev/lake/plugins/jira/models"
)
-var ErrNotFoundIssue = errors.New("not found the issue")
+var ErrNotFoundResource = errors.New("not found the resource")
type JiraApiRemotelink struct {
Id uint64
@@ -91,7 +91,7 @@
updated := jiraIssue.Updated
err = issueScheduler.Submit(func() error {
err = collector(source, jiraApiClient, issueId)
- if err == ErrNotFoundIssue {
+ if err == ErrNotFoundResource {
return nil
}
if err != nil {
@@ -123,7 +123,7 @@
return err
}
if res.StatusCode == http.StatusNotFound {
- return ErrNotFoundIssue
+ return ErrNotFoundResource
}
apiRemotelinks := &JiraApiRemotelinksResponse{}
err = core.UnmarshalResponse(res, apiRemotelinks)
diff --git a/plugins/jira/tasks/jira_server_api_client.go b/plugins/jira/tasks/jira_server_api_client.go
index 19507ce..9a3b621 100644
--- a/plugins/jira/tasks/jira_server_api_client.go
+++ b/plugins/jira/tasks/jira_server_api_client.go
@@ -78,7 +78,7 @@
func (v8 *ServerVersion8) collectRemotelinksByIssueId(source *models.JiraSource, jiraApiClient *JiraApiClient, issueId uint64) error {
var transformer v8models.RemoteLink
err := v8.Get(fmt.Sprintf("api/2/issue/%d/remotelink", issueId), v8.newHandlerWithIssueId(source.ID, issueId, transformer))
- if err != nil {
+ if err != nil && err != ErrNotFoundResource {
logger.Error("collect remotelink", err)
}
return nil
@@ -112,7 +112,7 @@
query.Set("jql", jql)
query.Set("expand", "changelog")
handler := func(resp *http.Response) (int, error) {
- return 0, v8.issueHandle(ctx, source, resp)
+ return 0, v8.issueHandle(ctx, boardId, source, resp)
}
err := v8.FetchPages(fmt.Sprintf("agile/1.0/board/%d/issue", boardId), &query, handler)
if err != nil {
@@ -121,7 +121,7 @@
return nil
}
-func (v8 *ServerVersion8) issueHandle(ctx context.Context, source *models.JiraSource, resp *http.Response) error {
+func (v8 *ServerVersion8) issueHandle(ctx context.Context, boardId uint64, source *models.JiraSource, resp *http.Response) error {
blob, err := ioutil.ReadAll(resp.Body)
if err != nil {
logger.Error("issueHandle read response body", err)
@@ -143,6 +143,7 @@
return nil
}
var jiraIssues []*models.JiraIssue
+ var boardIssues []*models.JiraBoardIssue
for _, apiIssue := range issues {
sprints, issue, needCollectWorklog, worklogs, changelogs, changelogItems := apiIssue.ExtractEntities(source.ID, source.StoryPointField)
for _, sprintId := range sprints {
@@ -160,6 +161,8 @@
}
}
jiraIssues = append(jiraIssues, issue)
+ boardIssue := &models.JiraBoardIssue{SourceId: source.ID, BoardId: boardId, IssueId: apiIssue.ID}
+ boardIssues = append(boardIssues, boardIssue)
if needCollectWorklog {
err = v8.collectWorklog(source.ID, issue.IssueId)
} else {
@@ -185,6 +188,11 @@
logger.Error("jira collect issues: save jiraIssues failed", err)
return err
}
+ err = v8.db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(boardIssues, BatchSize).Error
+ if err != nil {
+ logger.Error("jira collect issues: save board issues failed", err)
+ return err
+ }
return nil
}
@@ -265,6 +273,9 @@
func (v8 *ServerVersion8) newHandler(sourceId uint64, transformer v8models.Transformer) func(resp *http.Response) (int, error) {
return func(resp *http.Response) (int, error) {
+ if resp.StatusCode == http.StatusNotFound {
+ return 0, ErrNotFoundResource
+ }
blob, err := ioutil.ReadAll(resp.Body)
if err != nil {
logger.Error("handler factory read response body", err)
@@ -297,6 +308,9 @@
func (v8 *ServerVersion8) newHandlerWithIssueId(sourceId, issueId uint64, transformer v8models.TransformerWithIssueId) func(resp *http.Response) (int, error) {
return func(resp *http.Response) (int, error) {
+ if resp.StatusCode == http.StatusNotFound {
+ return 0, ErrNotFoundResource
+ }
blob, err := ioutil.ReadAll(resp.Body)
if err != nil {
logger.Error("handler factory read response body", err)
diff --git a/plugins/jira/tasks/jira_sprint_converter.go b/plugins/jira/tasks/jira_sprint_converter.go
index 4290519..92c4ec1 100644
--- a/plugins/jira/tasks/jira_sprint_converter.go
+++ b/plugins/jira/tasks/jira_sprint_converter.go
@@ -27,6 +27,7 @@
domainBoardId := didgen.NewDomainIdGenerator(&jiraModels.JiraBoard{}).Generate(sourceId, boardId)
sprintIdGen := didgen.NewDomainIdGenerator(&jiraModels.JiraSprint{})
issueIdGen := didgen.NewDomainIdGenerator(&jiraModels.JiraIssue{})
+ boardIdGen := didgen.NewDomainIdGenerator(&jiraModels.JiraBoard{})
// iterate all rows
for cursor.Next() {
var jiraSprint jiraModels.JiraSprint
@@ -42,6 +43,7 @@
StartedDate: jiraSprint.StartDate,
EndedDate: jiraSprint.EndDate,
CompletedDate: jiraSprint.CompleteDate,
+ OriginBoardID: boardIdGen.Generate(sourceId, jiraSprint.OriginBoardID),
}
err = lakeModels.Db.Clauses(clause.OnConflict{UpdateAll: true}).Create(sprint).Error
if err != nil {
diff --git a/plugins/jira/tasks/v8models/issue.go b/plugins/jira/tasks/v8models/issue.go
index b3c471e..5621bbe 100644
--- a/plugins/jira/tasks/v8models/issue.go
+++ b/plugins/jira/tasks/v8models/issue.go
@@ -72,28 +72,14 @@
Name string `json:"name"`
ID uint64 `json:"id,string"`
} `json:"priority"`
- Labels []interface{} `json:"labels"`
- Timeestimate interface{} `json:"timeestimate"`
- Aggregatetimeoriginalestimate interface{} `json:"aggregatetimeoriginalestimate"`
- Versions []interface{} `json:"versions"`
- Issuelinks []interface{} `json:"issuelinks"`
- Assignee *struct {
- Self string `json:"self"`
- Name string `json:"name"`
- Key string `json:"key"`
- EmailAddress string `json:"emailAddress"`
- AvatarUrls struct {
- Four8X48 string `json:"48x48"`
- Two4X24 string `json:"24x24"`
- One6X16 string `json:"16x16"`
- Three2X32 string `json:"32x32"`
- } `json:"avatarUrls"`
- DisplayName string `json:"displayName"`
- Active bool `json:"active"`
- TimeZone string `json:"timeZone"`
- } `json:"assignee"`
- Updated core.Iso8601Time `json:"updated"`
- Status struct {
+ Labels []interface{} `json:"labels"`
+ Timeestimate interface{} `json:"timeestimate"`
+ Aggregatetimeoriginalestimate interface{} `json:"aggregatetimeoriginalestimate"`
+ Versions []interface{} `json:"versions"`
+ Issuelinks []interface{} `json:"issuelinks"`
+ Assignee *User `json:"assignee"`
+ Updated core.Iso8601Time `json:"updated"`
+ Status struct {
Self string `json:"self"`
Description string `json:"description"`
IconURL string `json:"iconUrl"`
@@ -115,26 +101,12 @@
RemainingEstimateSeconds int64 `json:"remainingEstimateSeconds"`
TimeSpentSeconds int `json:"timeSpentSeconds"`
} `json:"timetracking"`
- Archiveddate interface{} `json:"archiveddate"`
- Aggregatetimeestimate *int64 `json:"aggregatetimeestimate"`
- Summary string `json:"summary"`
- Creator struct {
- Self string `json:"self"`
- Name string `json:"name"`
- Key string `json:"key"`
- EmailAddress string `json:"emailAddress"`
- AvatarUrls struct {
- Four8X48 string `json:"48x48"`
- Two4X24 string `json:"24x24"`
- One6X16 string `json:"16x16"`
- Three2X32 string `json:"32x32"`
- } `json:"avatarUrls"`
- DisplayName string `json:"displayName"`
- Active bool `json:"active"`
- TimeZone string `json:"timeZone"`
- } `json:"creator"`
- Subtasks []interface{} `json:"subtasks"`
- Reporter struct {
+ Archiveddate interface{} `json:"archiveddate"`
+ Aggregatetimeestimate *int64 `json:"aggregatetimeestimate"`
+ Summary string `json:"summary"`
+ Creator User `json:"creator"`
+ Subtasks []interface{} `json:"subtasks"`
+ Reporter struct {
Self string `json:"self"`
Name string `json:"name"`
Key string `json:"key"`
@@ -191,7 +163,7 @@
StatusName: i.Fields.Status.Name,
StatusKey: i.Fields.Status.StatusCategory.Key,
ResolutionDate: i.Fields.Resolutiondate.ToNullableTime(),
- CreatorAccountId: i.Fields.Creator.EmailAddress,
+ CreatorAccountId: i.Fields.Creator.getAccountId(),
CreatorDisplayName: i.Fields.Creator.DisplayName,
Created: i.Fields.Created.ToTime(),
Updated: i.Fields.Updated.ToTime(),
@@ -200,7 +172,7 @@
result.EpicKey = i.Fields.Epic.Key
}
if i.Fields.Assignee != nil {
- result.AssigneeAccountId = i.Fields.Assignee.EmailAddress
+ result.AssigneeAccountId = i.Fields.Assignee.getAccountId()
result.AssigneeDisplayName = i.Fields.Assignee.DisplayName
}
if i.Fields.Priority != nil {
diff --git a/plugins/jira/tasks/v8models/remotelink.go b/plugins/jira/tasks/v8models/remotelink.go
index 8069794..7cab0cf 100644
--- a/plugins/jira/tasks/v8models/remotelink.go
+++ b/plugins/jira/tasks/v8models/remotelink.go
@@ -2,6 +2,7 @@
import (
"encoding/json"
+ "gorm.io/datatypes"
"github.com/merico-dev/lake/plugins/jira/models"
)
@@ -34,7 +35,7 @@
} `json:"object"`
}
-func (r RemoteLink) toToolLayer(sourceId, issueId uint64) *models.JiraRemotelink {
+func (r RemoteLink) toToolLayer(sourceId, issueId uint64, raw json.RawMessage) *models.JiraRemotelink {
return &models.JiraRemotelink{
SourceId: sourceId,
RemotelinkId: r.ID,
@@ -42,18 +43,24 @@
Self: r.Self,
Title: r.Object.Title,
Url: r.Object.URL,
+ RawJson: datatypes.JSON(raw),
}
}
func (RemoteLink) FromAPI(sourceId, issueId uint64, raw json.RawMessage) (interface{}, error) {
- var vv []RemoteLink
- err := json.Unmarshal(raw, &vv)
+ var msgs []json.RawMessage
+ err := json.Unmarshal(raw, &msgs)
if err != nil {
return nil, err
}
- list := make([]*models.JiraRemotelink, len(vv))
- for i, item := range vv {
- list[i] = item.toToolLayer(sourceId, issueId)
+ var list []*models.JiraRemotelink
+ for _, msg := range msgs {
+ var remoteLink RemoteLink
+ err = json.Unmarshal(msg, &remoteLink)
+ if err != nil {
+ return nil, err
+ }
+ list = append(list, remoteLink.toToolLayer(sourceId, issueId, msg))
}
return list, nil
}
diff --git a/plugins/jira/tasks/v8models/user.go b/plugins/jira/tasks/v8models/user.go
index d463f51..c5412bc 100644
--- a/plugins/jira/tasks/v8models/user.go
+++ b/plugins/jira/tasks/v8models/user.go
@@ -5,6 +5,7 @@
Key string `json:"key"`
Name string `json:"name"`
EmailAddress string `json:"emailAddress"`
+ AccountId string `json:"accountId"`
AvatarUrls struct {
Four8X48 string `json:"48x48"`
Two4X24 string `json:"24x24"`
@@ -17,3 +18,13 @@
TimeZone string `json:"timeZone"`
Locale string `json:"locale"`
}
+
+func (u *User) getAccountId() string {
+ if u == nil {
+ return ""
+ }
+ if u.AccountId != "" {
+ return u.AccountId
+ }
+ return u.EmailAddress
+}
