#1228 sync model (#1241)
1. print error log when parse task
2. print task count info when load task
Co-authored-by: aseTo2016 <tys201193111>
diff --git a/datasource/manager.go b/datasource/manager.go
index df00229..2e7c91d 100644
--- a/datasource/manager.go
+++ b/datasource/manager.go
@@ -66,7 +66,10 @@
return err
}
// init eventbase
- err = datasource.Init(opts.Kind)
+ err = datasource.Init(&datasource.Config{
+ Kind: opts.Kind,
+ Logger: log.Logger,
+ })
return err
}
diff --git a/eventbase/datasource/etcd/task/task_dao.go b/eventbase/datasource/etcd/task/task_dao.go
index 71bc707..7f45507 100644
--- a/eventbase/datasource/etcd/task/task_dao.go
+++ b/eventbase/datasource/etcd/task/task_dao.go
@@ -21,11 +21,12 @@
"context"
"encoding/json"
- "github.com/go-chassis/cari/sync"
- "github.com/little-cui/etcdadpt"
-
"github.com/apache/servicecomb-service-center/eventbase/datasource"
"github.com/apache/servicecomb-service-center/eventbase/datasource/etcd/key"
+
+ "github.com/go-chassis/cari/sync"
+ "github.com/go-chassis/openlog"
+ "github.com/little-cui/etcdadpt"
)
type Dao struct {
@@ -95,6 +96,7 @@
task := sync.Task{}
err := json.Unmarshal(kv.Value, &task)
if err != nil {
+ datasource.Logger().Error("unmarshal task failed", openlog.WithErr(err))
continue
}
if !filterMatch(&task, opts) {
diff --git a/eventbase/datasource/manager.go b/eventbase/datasource/manager.go
index e267fc3..dbcf387 100644
--- a/eventbase/datasource/manager.go
+++ b/eventbase/datasource/manager.go
@@ -19,13 +19,21 @@
import (
"fmt"
+
+ "github.com/go-chassis/openlog"
)
var (
dataSourceInst DataSource
plugins = make(map[string]dataSourceEngine)
+
+ logger openlog.Logger
)
+func Logger() openlog.Logger {
+ return logger
+}
+
type dataSourceEngine func() DataSource
func GetDataSource() DataSource {
@@ -36,11 +44,20 @@
plugins[name] = engineFunc
}
-func Init(kind string) error {
- f, ok := plugins[kind]
+type Config struct {
+ Kind string
+ Logger openlog.Logger
+}
+
+func Init(c *Config) error {
+ f, ok := plugins[c.Kind]
if !ok {
- return fmt.Errorf("do not support %s", kind)
+ return fmt.Errorf("do not support %s", c.Kind)
}
+ if c.Logger != nil {
+ logger = c.Logger
+ }
+
dataSourceInst = f()
return nil
}
diff --git a/eventbase/datasource/manager_test.go b/eventbase/datasource/manager_test.go
index 53a0544..5ba493d 100644
--- a/eventbase/datasource/manager_test.go
+++ b/eventbase/datasource/manager_test.go
@@ -20,15 +20,18 @@
import (
"testing"
+ "github.com/apache/servicecomb-service-center/eventbase/test"
"github.com/stretchr/testify/assert"
"github.com/apache/servicecomb-service-center/eventbase/datasource"
- "github.com/apache/servicecomb-service-center/eventbase/test"
)
func TestInit(t *testing.T) {
t.Run("init config should pass with no error", func(t *testing.T) {
- err := datasource.Init(test.DBKind)
+ err := datasource.Init(&datasource.Config{
+ Kind: test.DBKind,
+ Logger: nil,
+ })
assert.Nil(t, err)
assert.NotNil(t, datasource.GetDataSource())
})
diff --git a/eventbase/go.mod b/eventbase/go.mod
index 2f56f74..a0586ab 100644
--- a/eventbase/go.mod
+++ b/eventbase/go.mod
@@ -3,6 +3,7 @@
require (
github.com/go-chassis/cari v0.5.1-0.20220119150556-8ae374a2649d
github.com/go-chassis/go-archaius v1.5.4
+ github.com/go-chassis/openlog v1.1.3
github.com/little-cui/etcdadpt v0.3.2
github.com/stretchr/testify v1.7.0
go.mongodb.org/mongo-driver v1.4.2
@@ -22,7 +23,6 @@
github.com/fsnotify/fsnotify v1.4.7 // indirect
github.com/go-chassis/foundation v0.4.0 // indirect
github.com/go-chassis/go-chassis/v2 v2.3.0 // indirect
- github.com/go-chassis/openlog v1.1.3 // indirect
github.com/go-stack/stack v1.8.0 // indirect
github.com/gofrs/uuid v4.0.0+incompatible // indirect
github.com/gogo/protobuf v1.3.2 // indirect
diff --git a/eventbase/go.sum b/eventbase/go.sum
index c67fcb7..2bee151 100644
--- a/eventbase/go.sum
+++ b/eventbase/go.sum
@@ -112,8 +112,6 @@
github.com/go-chassis/cari v0.0.0-20201210041921-7b6fbef2df11/go.mod h1:MgtsEI0AM4Ush6Lyw27z9Gk4nQ/8GWTSXrFzupawWDM=
github.com/go-chassis/cari v0.4.0/go.mod h1:av/19fqwEP4eOC8unL/z67AAbFDwXUCko6SKa4Avrd8=
github.com/go-chassis/cari v0.5.0/go.mod h1:av/19fqwEP4eOC8unL/z67AAbFDwXUCko6SKa4Avrd8=
-github.com/go-chassis/cari v0.5.1-0.20220119105129-dc7bd5491b49 h1:Qy+Q90kqzVrPTRncrkruPDFQOPssoJ86QjnesJMMNLc=
-github.com/go-chassis/cari v0.5.1-0.20220119105129-dc7bd5491b49/go.mod h1:tKTzguHTGohMCgkcWNZWtA4TwfcsJrIXpfYxsQtb7uw=
github.com/go-chassis/cari v0.5.1-0.20220119150556-8ae374a2649d h1:RtBn1T7KmJM1j1+NlBFqaKJWPWPDde9adDQMFHCKMbU=
github.com/go-chassis/cari v0.5.1-0.20220119150556-8ae374a2649d/go.mod h1:tKTzguHTGohMCgkcWNZWtA4TwfcsJrIXpfYxsQtb7uw=
github.com/go-chassis/foundation v0.2.2-0.20201210043510-9f6d3de40234/go.mod h1:2PjwqpVwYEVaAldl5A58a08viH8p27pNeYaiE3ZxOBA=
@@ -121,7 +119,6 @@
github.com/go-chassis/foundation v0.3.0/go.mod h1:2PjwqpVwYEVaAldl5A58a08viH8p27pNeYaiE3ZxOBA=
github.com/go-chassis/foundation v0.4.0 h1:z0xETnSxF+vRXWjoIhOdzt6rywjZ4sB++utEl4YgWEY=
github.com/go-chassis/foundation v0.4.0/go.mod h1:6NsIUaHghTFRGfCBcZN011zl196F6OR5QvD9N+P4oWU=
-github.com/go-chassis/go-archaius v1.5.1 h1:1FrNyzzmD6o6BIjPF8uQ4Cc+u7qYIgQTpDk8uopBqfo=
github.com/go-chassis/go-archaius v1.5.1/go.mod h1:QPwvvtBxvwiC48rmydoAqxopqOr93RCQ6syWsIkXPXQ=
github.com/go-chassis/go-archaius v1.5.4 h1:5mGXmBNfYnP8oc5KmXyds3kNMtWqlQ4VQKeyF2lvWs4=
github.com/go-chassis/go-archaius v1.5.4/go.mod h1:ZyAZzEMPyEsyCP7KLYB/48eX/dDj8CzO/CL87P80AOc=
diff --git a/eventbase/service/task/task_svc_test.go b/eventbase/service/task/task_svc_test.go
index b3e85e5..004bfcc 100644
--- a/eventbase/service/task/task_svc_test.go
+++ b/eventbase/service/task/task_svc_test.go
@@ -32,7 +32,10 @@
)
func init() {
- err := datasource.Init(test.DBKind)
+ err := datasource.Init(&datasource.Config{
+ Kind: test.DBKind,
+ Logger: nil,
+ })
if err != nil {
panic(err)
}
diff --git a/eventbase/service/tombstone/tombstone_svc_test.go b/eventbase/service/tombstone/tombstone_svc_test.go
index 1257445..9443a15 100644
--- a/eventbase/service/tombstone/tombstone_svc_test.go
+++ b/eventbase/service/tombstone/tombstone_svc_test.go
@@ -21,17 +21,20 @@
"context"
"testing"
+ "github.com/apache/servicecomb-service-center/eventbase/test"
"github.com/go-chassis/cari/sync"
"github.com/stretchr/testify/assert"
"github.com/apache/servicecomb-service-center/eventbase/datasource"
"github.com/apache/servicecomb-service-center/eventbase/model"
"github.com/apache/servicecomb-service-center/eventbase/service/tombstone"
- "github.com/apache/servicecomb-service-center/eventbase/test"
)
func init() {
- err := datasource.Init(test.DBKind)
+ err := datasource.Init(&datasource.Config{
+ Kind: test.DBKind,
+ Logger: nil,
+ })
if err != nil {
panic(err)
}
diff --git a/syncer/service/task/manager.go b/syncer/service/task/manager.go
index 0fdc69c..270a963 100644
--- a/syncer/service/task/manager.go
+++ b/syncer/service/task/manager.go
@@ -31,8 +31,6 @@
}
func work() {
- initDatabase()
-
dl := DistributedLock{
key: taskName,
heartbeatDuration: heartbeatInternal,
@@ -166,9 +164,11 @@
}
noHandleTasks := make([]*carisync.Task, 0, len(tasks))
+ skipTaskIDs := make([]string, 0, len(tasks))
for _, t := range tasks {
_, ok := m.cache.Load(t.ID)
if ok {
+ skipTaskIDs = append(skipTaskIDs, t.ID)
continue
}
m.cache.Store(t.ID, t)
@@ -176,7 +176,8 @@
noHandleTasks = append(noHandleTasks, t)
}
- log.Info(fmt.Sprintf("load task count %d", len(noHandleTasks)))
+ log.Info(fmt.Sprintf("load task raw count %d, to handle count %d, skip ids %v",
+ len(tasks), len(noHandleTasks), skipTaskIDs))
return noHandleTasks, nil
}
diff --git a/syncer/service/task/task.go b/syncer/service/task/task.go
index ae489e7..4c9d2cd 100644
--- a/syncer/service/task/task.go
+++ b/syncer/service/task/task.go
@@ -3,24 +3,11 @@
import (
"context"
- "github.com/apache/servicecomb-service-center/eventbase/datasource"
"github.com/apache/servicecomb-service-center/eventbase/model"
servicetask "github.com/apache/servicecomb-service-center/eventbase/service/task"
- "github.com/apache/servicecomb-service-center/pkg/log"
- serverconfig "github.com/apache/servicecomb-service-center/server/config"
-
carisync "github.com/go-chassis/cari/sync"
)
-func initDatabase() {
- kind := serverconfig.GetString("registry.kind", "",
- serverconfig.WithStandby("registry_plugin"))
-
- if err := datasource.Init(kind); err != nil {
- log.Fatal("init datasource failed", err)
- }
-}
-
func ListTask(ctx context.Context) ([]*carisync.Task, error) {
return servicetask.List(ctx, &model.ListTaskRequest{})
}
diff --git a/syncer/service/tombstone/tombstone_test.go b/syncer/service/tombstone/tombstone_test.go
index 4c44cc4..d01a626 100644
--- a/syncer/service/tombstone/tombstone_test.go
+++ b/syncer/service/tombstone/tombstone_test.go
@@ -32,7 +32,10 @@
)
func init() {
- err := datasource.Init(test.DBKind)
+ err := datasource.Init(&datasource.Config{
+ Kind: test.DBKind,
+ Logger: nil,
+ })
if err != nil {
panic(err)
}
diff --git a/test/test.go b/test/test.go
index 2300c8b..ef92a0e 100644
--- a/test/test.go
+++ b/test/test.go
@@ -71,7 +71,10 @@
Timeout: 10 * time.Second,
})
- _ = edatasource.Init(kind)
+ _ = edatasource.Init(&edatasource.Config{
+ Kind: kind,
+ Logger: nil,
+ })
_ = registry.SelfRegister(context.Background())
}
