Merge branch 'release-v0.21' into lw-cp0223
diff --git a/backend/plugins/zentao/models/story.go b/backend/plugins/zentao/models/story.go
index 661a626..42956b7 100644
--- a/backend/plugins/zentao/models/story.go
+++ b/backend/plugins/zentao/models/story.go
@@ -22,60 +22,60 @@
)
type ZentaoStoryRes struct {
- ID int64 `json:"id"`
- Vision string `json:"vision"`
- Parent int64 `json:"parent"`
- Product int64 `json:"product"`
- Branch int `json:"branch"`
- Module int `json:"module"`
- Plan string `json:"plan"`
- Source string `json:"source"`
- SourceNote string `json:"sourceNote"`
- FromBug int `json:"fromBug"`
- Feedback int `json:"feedback"`
- Title string `json:"title"`
- Keywords string `json:"keywords"`
- Type string `json:"type"`
- Category string `json:"category"`
- Pri int `json:"pri"`
- Estimate float64 `json:"estimate"`
- Status string `json:"status"`
- SubStatus string `json:"subStatus"`
- Color string `json:"color"`
- Stage string `json:"stage"`
- Mailto []interface{} `json:"mailto"`
- Lib int `json:"lib"`
- FromStory int64 `json:"fromStory"`
- FromVersion int `json:"fromVersion"`
- OpenedBy *ApiAccount `json:"openedBy"`
- OpenedDate *common.Iso8601Time `json:"openedDate"`
- AssignedTo *ApiAccount `json:"assignedTo"`
- AssignedDate *common.Iso8601Time `json:"assignedDate"`
- ApprovedDate *common.Iso8601Time `json:"approvedDate"`
- LastEditedBy *ApiAccount `json:"lastEditedBy"`
- LastEditedDate *common.Iso8601Time `json:"lastEditedDate"`
- ChangedBy string `json:"changedBy"`
- ChangedDate *common.Iso8601Time `json:"changedDate"`
- ReviewedBy *ApiAccount `json:"reviewedBy"`
- ReviewedDate *common.Iso8601Time `json:"reviewedDate"`
- ClosedBy *ApiAccount `json:"closedBy"`
- ClosedDate *common.Iso8601Time `json:"closedDate"`
- ClosedReason string `json:"closedReason"`
- ActivatedDate *common.Iso8601Time `json:"activatedDate"`
- ToBug int `json:"toBug"`
- ChildStories string `json:"childStories"`
- LinkStories string `json:"linkStories"`
- LinkRequirements string `json:"linkRequirements"`
- DuplicateStory int64 `json:"duplicateStory"`
- Version int `json:"version"`
- StoryChanged string `json:"storyChanged"`
- FeedbackBy string `json:"feedbackBy"`
- NotifyEmail string `json:"notifyEmail"`
- URChanged string `json:"URChanged"`
- Deleted bool `json:"deleted"`
- PriOrder string `json:"priOrder"`
- PlanTitle string `json:"planTitle"`
- ProductStatus string `json:"productStatus"`
+ ID int64 `json:"id"`
+ Vision string `json:"vision"`
+ Parent int64 `json:"parent"`
+ Product int64 `json:"product"`
+ Branch int `json:"branch"`
+ Module int `json:"module"`
+ Plan string `json:"plan"`
+ Source string `json:"source"`
+ SourceNote string `json:"sourceNote"`
+ FromBug int `json:"fromBug"`
+ Feedback int `json:"feedback"`
+ Title string `json:"title"`
+ Keywords string `json:"keywords"`
+ Type string `json:"type"`
+ Category string `json:"category"`
+ Pri int `json:"pri"`
+ Estimate float64 `json:"estimate"`
+ Status string `json:"status"`
+ SubStatus string `json:"subStatus"`
+ Color string `json:"color"`
+ Stage string `json:"stage"`
+ Mailto []interface{} `json:"mailto"`
+ Lib int `json:"lib"`
+ FromStory int64 `json:"fromStory"`
+ FromVersion int `json:"fromVersion"`
+ OpenedBy *ApiAccount `json:"openedBy"`
+ OpenedDate *common.Iso8601Time `json:"openedDate"`
+ AssignedTo *ApiAccount `json:"assignedTo"`
+ AssignedDate *common.Iso8601Time `json:"assignedDate"`
+ ApprovedDate *common.Iso8601Time `json:"approvedDate"`
+ LastEditedBy *ApiAccount `json:"lastEditedBy"`
+ LastEditedDate *common.Iso8601Time `json:"lastEditedDate"`
+ ChangedBy string `json:"changedBy"`
+ ChangedDate *common.Iso8601Time `json:"changedDate"`
+ ReviewedBy *ApiAccount `json:"reviewedBy"`
+ ReviewedDate *common.Iso8601Time `json:"reviewedDate"`
+ ClosedBy *ApiAccount `json:"closedBy"`
+ ClosedDate *common.Iso8601Time `json:"closedDate"`
+ ClosedReason string `json:"closedReason"`
+ ActivatedDate *common.Iso8601Time `json:"activatedDate"`
+ ToBug int `json:"toBug"`
+ ChildStories string `json:"childStories"`
+ LinkStories string `json:"linkStories"`
+ LinkRequirements string `json:"linkRequirements"`
+ DuplicateStory int64 `json:"duplicateStory"`
+ Version int `json:"version"`
+ StoryChanged string `json:"storyChanged"`
+ FeedbackBy string `json:"feedbackBy"`
+ NotifyEmail string `json:"notifyEmail"`
+ URChanged string `json:"URChanged"`
+ Deleted bool `json:"deleted"`
+ PriOrder *common.StringFloat64 `json:"priOrder"`
+ PlanTitle string `json:"planTitle"`
+ ProductStatus string `json:"productStatus"`
}
type ZentaoStory struct {
diff --git a/backend/plugins/zentao/models/task.go b/backend/plugins/zentao/models/task.go
index 8bbc5fe..494d64a 100644
--- a/backend/plugins/zentao/models/task.go
+++ b/backend/plugins/zentao/models/task.go
@@ -83,12 +83,12 @@
} `json:"latestStoryVersion"`
StoryStatus interface {
} `json:"storyStatus"`
- AssignedToRealName string `json:"assignedToRealName"`
- PriOrder string `json:"priOrder"`
- Children []*ZentaoTaskRes `json:"children"`
- Delay int `json:"delay"`
- NeedConfirm bool `json:"needConfirm"`
- Progress float64 `json:"progress"`
+ AssignedToRealName string `json:"assignedToRealName"`
+ PriOrder *common.StringFloat64 `json:"priOrder"`
+ Children []*ZentaoTaskRes `json:"children"`
+ Delay int `json:"delay"`
+ NeedConfirm bool `json:"needConfirm"`
+ Progress float64 `json:"progress"`
}
func (zentaoTaskRes ZentaoTaskRes) ToJsonRawMessage() (json.RawMessage, error) {
diff --git a/backend/plugins/zentao/tasks/story_extractor.go b/backend/plugins/zentao/tasks/story_extractor.go
index 7f94c4f..1d00dcf 100644
--- a/backend/plugins/zentao/tasks/story_extractor.go
+++ b/backend/plugins/zentao/tasks/story_extractor.go
@@ -120,7 +120,7 @@
NotifyEmail: res.NotifyEmail,
URChanged: res.URChanged,
Deleted: res.Deleted,
- PriOrder: res.PriOrder,
+ PriOrder: res.PriOrder.String(),
PlanTitle: res.PlanTitle,
Url: row.Url,
}
diff --git a/backend/plugins/zentao/tasks/task_extractor.go b/backend/plugins/zentao/tasks/task_extractor.go
index d18701c..422e5ee 100644
--- a/backend/plugins/zentao/tasks/task_extractor.go
+++ b/backend/plugins/zentao/tasks/task_extractor.go
@@ -146,7 +146,7 @@
StoryTitle: res.StoryTitle,
LatestStoryVersion: 0,
AssignedToRealName: res.AssignedToRealName,
- PriOrder: res.PriOrder,
+ PriOrder: res.PriOrder.String(),
NeedConfirm: res.NeedConfirm,
Progress: res.Progress,
Url: url,
diff --git a/backend/server/api/api.go b/backend/server/api/api.go
index a4894cf..e04e9e6 100644
--- a/backend/server/api/api.go
+++ b/backend/server/api/api.go
@@ -166,7 +166,7 @@
}
// Start the server
- err = router.Run(fmt.Sprintf("0.0.0.0:%d", portNum))
+ err = router.Run(fmt.Sprintf(":%d", portNum))
if err != nil {
panic(err)
}
diff --git a/backend/server/api/middlewares.go b/backend/server/api/middlewares.go
index 9e274c2..32ec5e8 100644
--- a/backend/server/api/middlewares.go
+++ b/backend/server/api/middlewares.go
@@ -20,6 +20,7 @@
import (
"encoding/base64"
"fmt"
+ "github.com/apache/incubator-devlake/core/log"
"net/http"
"regexp"
"strings"
@@ -95,11 +96,13 @@
}
}
+type apiBody struct {
+ Success bool `json:"success"`
+ Message string `json:"message"`
+}
+
func RestAuthentication(router *gin.Engine, basicRes context.BasicRes) gin.HandlerFunc {
- type ApiBody struct {
- Success bool `json:"success"`
- Message string `json:"message"`
- }
+
db := basicRes.GetDal()
logger := basicRes.GetLogger()
if db == nil {
@@ -115,88 +118,97 @@
return
}
path = strings.TrimPrefix(path, "/rest")
-
authHeader := c.GetHeader("Authorization")
- if authHeader == "" {
+ ok := CheckAuthorizationHeader(c, logger, db, apiKeyHelper, authHeader, path)
+ if !ok {
+ return
+ } else {
+ router.HandleContext(c)
c.Abort()
- c.JSON(http.StatusUnauthorized, &ApiBody{
- Success: false,
- Message: "token is missing",
- })
return
}
- apiKeyStr := strings.TrimPrefix(authHeader, "Bearer ")
- if apiKeyStr == authHeader || apiKeyStr == "" {
- c.Abort()
- c.JSON(http.StatusUnauthorized, &ApiBody{
- Success: false,
- Message: "token is not present or malformed",
- })
- return
- }
+ }
+}
- hashedApiKey, err := apiKeyHelper.DigestToken(apiKeyStr)
- if err != nil {
- logger.Error(err, "DigestToken")
- c.Abort()
- c.JSON(http.StatusInternalServerError, &ApiBody{
+func CheckAuthorizationHeader(c *gin.Context, logger log.Logger, db dal.Dal, apiKeyHelper *apikeyhelper.ApiKeyHelper, authHeader, path string) bool {
+ if authHeader == "" {
+ c.Abort()
+ c.JSON(http.StatusUnauthorized, &apiBody{
+ Success: false,
+ Message: "token is missing",
+ })
+ return false
+ }
+ apiKeyStr := strings.TrimPrefix(authHeader, "Bearer ")
+ if apiKeyStr == authHeader || apiKeyStr == "" {
+ c.Abort()
+ c.JSON(http.StatusUnauthorized, &apiBody{
+ Success: false,
+ Message: "token is not present or malformed",
+ })
+ return false
+ }
+
+ hashedApiKey, err := apiKeyHelper.DigestToken(apiKeyStr)
+ if err != nil {
+ logger.Error(err, "DigestToken")
+ c.Abort()
+ c.JSON(http.StatusInternalServerError, &apiBody{
+ Success: false,
+ Message: err.Error(),
+ })
+ return false
+ }
+
+ apiKey, err := apiKeyHelper.GetApiKey(nil, dal.Where("api_key = ?", hashedApiKey))
+ if err != nil {
+ c.Abort()
+ if db.IsErrorNotFound(err) {
+ c.JSON(http.StatusForbidden, &apiBody{
+ Success: false,
+ Message: "api key is invalid",
+ })
+ } else {
+ logger.Error(err, "query api key from db")
+ c.JSON(http.StatusInternalServerError, &apiBody{
Success: false,
Message: err.Error(),
})
- return
}
-
- apiKey, err := apiKeyHelper.GetApiKey(nil, dal.Where("api_key = ?", hashedApiKey))
- if err != nil {
- c.Abort()
- if db.IsErrorNotFound(err) {
- c.JSON(http.StatusForbidden, &ApiBody{
- Success: false,
- Message: "api key is invalid",
- })
- } else {
- logger.Error(err, "query api key from db")
- c.JSON(http.StatusInternalServerError, &ApiBody{
- Success: false,
- Message: err.Error(),
- })
- }
- return
- }
-
- if apiKey.ExpiredAt != nil && time.Until(*apiKey.ExpiredAt) < 0 {
- c.Abort()
- c.JSON(http.StatusForbidden, &ApiBody{
- Success: false,
- Message: "api key has expired",
- })
- return
- }
- matched, matchErr := regexp.MatchString(apiKey.AllowedPath, path)
- if matchErr != nil {
- logger.Error(err, "regexp match path error")
- c.Abort()
- c.JSON(http.StatusInternalServerError, &ApiBody{
- Success: false,
- Message: matchErr.Error(),
- })
- return
- }
- if !matched {
- c.JSON(http.StatusForbidden, &ApiBody{
- Success: false,
- Message: "path doesn't match api key's scope",
- })
- return
- }
-
- logger.Info("redirect path: %s to: %s", c.Request.URL.Path, path)
- c.Request.URL.Path = path
- c.Set(common.USER, &common.User{
- Name: apiKey.Creator.Creator,
- Email: apiKey.Creator.CreatorEmail,
- })
- router.HandleContext(c)
- c.Abort()
+ return false
}
+
+ if apiKey.ExpiredAt != nil && time.Until(*apiKey.ExpiredAt) < 0 {
+ c.Abort()
+ c.JSON(http.StatusForbidden, &apiBody{
+ Success: false,
+ Message: "api key has expired",
+ })
+ return false
+ }
+ matched, matchErr := regexp.MatchString(apiKey.AllowedPath, path)
+ if matchErr != nil {
+ logger.Error(err, "regexp match path error")
+ c.Abort()
+ c.JSON(http.StatusInternalServerError, &apiBody{
+ Success: false,
+ Message: matchErr.Error(),
+ })
+ return false
+ }
+ if !matched {
+ c.JSON(http.StatusForbidden, &apiBody{
+ Success: false,
+ Message: "path doesn't match api key's scope",
+ })
+ return false
+ }
+
+ logger.Info("redirect path: %s to: %s", c.Request.URL.Path, path)
+ c.Request.URL.Path = path
+ c.Set(common.USER, &common.User{
+ Name: apiKey.Creator.Creator,
+ Email: apiKey.Creator.CreatorEmail,
+ })
+ return true
}
diff --git a/backend/server/services/blueprint.go b/backend/server/services/blueprint.go
index fa28508..9a8407e 100644
--- a/backend/server/services/blueprint.go
+++ b/backend/server/services/blueprint.go
@@ -68,22 +68,8 @@
// CreateBlueprint accepts a Blueprint instance and insert it to database
func CreateBlueprint(blueprint *models.Blueprint) errors.Error {
- err := validateBlueprintAndMakePlan(blueprint)
- if err != nil {
- return err
- }
- err = bpManager.SaveDbBlueprint(blueprint)
- if err != nil {
- return err
- }
- if err := SanitizeBlueprint(blueprint); err != nil {
- return errors.Convert(err)
- }
- err = ReloadBlueprints(cronManager)
- if err != nil {
- return errors.Internal.Wrap(err, "error reloading blueprints")
- }
- return nil
+ _, err := saveBlueprint(blueprint)
+ return err
}
// GetBlueprints returns a paginated list of Blueprints based on `query`
@@ -204,9 +190,9 @@
}
// reload schedule
- err = ReloadBlueprints(cronManager)
+ err = reloadBlueprint(blueprint)
if err != nil {
- return nil, errors.Internal.Wrap(err, "error reloading blueprints")
+ return nil, err
}
// done
return blueprint, nil
@@ -260,14 +246,10 @@
}
var blueprintReloadLock sync.Mutex
+var bpCronIdMap map[uint64]cron.EntryID
-// ReloadBlueprints FIXME ...
-func ReloadBlueprints(c *cron.Cron) (err errors.Error) {
- // preventing concurrent reloads. It would be better to use Table Lock , however, it requires massive refactor
- // like the `bpManager` must accept transaction. Use mutex as a temporary fix.
- blueprintReloadLock.Lock()
- defer blueprintReloadLock.Unlock()
-
+// ReloadBlueprints reloades cronjobs based on blueprints
+func ReloadBlueprints() (err errors.Error) {
enable := true
isManual := false
blueprints, _, err := bpManager.GetDbBlueprints(&services.GetBlueprintQuery{
@@ -277,27 +259,48 @@
if err != nil {
return err
}
- for _, e := range c.Entries() {
- c.Remove(e.ID)
+ for _, e := range cronManager.Entries() {
+ cronManager.Remove(e.ID)
}
- c.Stop()
+ cronManager.Stop()
+ bpCronIdMap = make(map[uint64]cron.EntryID, len(blueprints))
for _, blueprint := range blueprints {
- blueprintLog.Info("Add blueprint id:[%d] cronConfg[%s] to cron job", blueprint.ID, blueprint.CronConfig)
- blueprintJob := &BlueprintJob{
- Blueprint: blueprint,
- }
- if _, err := c.AddJob(blueprint.CronConfig, blueprintJob); err != nil {
- blueprintLog.Error(err, failToCreateCronJob)
- return errors.Default.Wrap(err, "created cron job failed")
+ err := reloadBlueprint(blueprint)
+ if err != nil {
+ return err
}
}
if len(blueprints) > 0 {
- c.Start()
+ cronManager.Start()
}
logger.Info("total %d blueprints were scheduled", len(blueprints))
return nil
}
+func reloadBlueprint(blueprint *models.Blueprint) errors.Error {
+ // preventing concurrent reloads. It would be better to use Table Lock , however, it requires massive refactor
+ // like the `bpManager` must accept transaction. Use mutex as a temporary fix.
+ blueprintReloadLock.Lock()
+ defer blueprintReloadLock.Unlock()
+
+ cronId, scheduled := bpCronIdMap[blueprint.ID]
+ if scheduled {
+ cronManager.Remove(cronId)
+ delete(bpCronIdMap, blueprint.ID)
+ logger.Info("removed blueprint %d from cronjobs, cron id: %v", blueprint.ID, cronId)
+ }
+ if blueprint.Enable && !blueprint.IsManual {
+ if cronId, err := cronManager.AddJob(blueprint.CronConfig, &BlueprintJob{blueprint}); err != nil {
+ blueprintLog.Error(err, failToCreateCronJob)
+ return errors.Default.Wrap(err, "created cron job failed")
+ } else {
+ bpCronIdMap[blueprint.ID] = cronId
+ logger.Info("added blueprint %d to cronjobs, cron id: %v, cron config: %s", blueprint.ID, cronId, blueprint.CronConfig)
+ }
+ }
+ return nil
+}
+
func createPipelineByBlueprint(blueprint *models.Blueprint, syncPolicy *models.SyncPolicy) (*models.Pipeline, errors.Error) {
var plan models.PipelinePlan
var err errors.Error
diff --git a/backend/server/services/pipeline.go b/backend/server/services/pipeline.go
index 1783c94..2a1ac8c 100644
--- a/backend/server/services/pipeline.go
+++ b/backend/server/services/pipeline.go
@@ -103,7 +103,7 @@
panic(err)
}
- err = ReloadBlueprints(cronManager)
+ err = ReloadBlueprints()
if err != nil {
panic(err)
}
