SCB-1613 a new API for user want to list key values by labels (#53)
diff --git a/.travis.yml b/.travis.yml
index d5a358a..a425a01 100755
--- a/.travis.yml
+++ b/.travis.yml
@@ -58,4 +58,4 @@
- cd $HOME/gopath/src/github.com/apache/servicecomb-kie
- go get github.com/mattn/goveralls
- go get golang.org/x/tools/cmd/cover
- - bash scripts/travis/unit_test.sh
+ - bash scripts/travis/unit_test.sh && $HOME/gopath/bin/goveralls -coverprofile=coverage.txt -service=travis-ci
diff --git a/README.md b/README.md
index f3bf240..81f9c15 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,8 @@
-# Apache-ServiceComb-Kie [](https://travis-ci.org/apache/servicecomb-kie?branch=master) [](https://www.apache.org/licenses/LICENSE-2.0.html)
+# Apache-ServiceComb-Kie
+[](https://travis-ci.org/apache/servicecomb-kie?branch=master)
+[](https://www.apache.org/licenses/LICENSE-2.0.html)
+[](https://coveralls.io/github/apache/servicecomb-kie?branch=master)
A service for configuration management in distributed system.
## Conceptions
diff --git a/client/adaptor/kie_client.go b/client/adaptor/kie_client.go
index c2d0137..a640e8a 100644
--- a/client/adaptor/kie_client.go
+++ b/client/adaptor/kie_client.go
@@ -60,15 +60,15 @@
var err error
var configurationsValue []*model.KVResponse
if len(labels) != 0 {
- configurationsValue, err = c.KieClient.SearchByLabels(context.TODO(), client.WithGetProject("default"), client.WithLabels(labels...))
+ configurationsValue, err = c.KieClient.Search(context.TODO(), client.WithGetProject("default"), client.WithLabels(labels...))
} else {
- configurationsValue, err = c.KieClient.SearchByLabels(context.TODO(), client.WithGetProject("default"), client.WithLabels(c.opts.Labels))
+ configurationsValue, err = c.KieClient.Search(context.TODO(), client.WithGetProject("default"), client.WithLabels(c.opts.Labels))
}
if err != nil {
openlogging.GetLogger().Errorf("Error in Querying the Response from Kie %s %#v", err.Error(), labels)
return nil, err
}
- openlogging.GetLogger().Debugf("KieClient SearchByLabels. %#v", labels)
+ openlogging.GetLogger().Debugf("KieClient Search. %#v", labels)
//Parse config result.
for _, docRes := range configurationsValue {
for _, docInfo := range docRes.Data {
diff --git a/client/adaptor/kie_client_test.go b/client/adaptor/kie_client_test.go
index 9813512..cfd6954 100644
--- a/client/adaptor/kie_client_test.go
+++ b/client/adaptor/kie_client_test.go
@@ -138,7 +138,7 @@
config.LabelService: "test",
})
//assert.Equal(t, resp.StatusCode, 404)
- assert.Equal(t, err.Error(), "delete 1 failed,http status [200 OK], body [[{\"label\":null,\"data\":null}]]")
+ assert.Equal(t, "delete 1 failed,http status [200 OK], body [[{\"data\":null}]]", err.Error())
// Shutdown the helper server gracefully
if err := helper.Shutdown(context.Background()); err != nil {
panic(err)
diff --git a/client/client.go b/client/client.go
index 342038b..51bca74 100644
--- a/client/client.go
+++ b/client/client.go
@@ -158,8 +158,8 @@
return kvs, nil
}
-//SearchByLabels get value by lables
-func (c *Client) SearchByLabels(ctx context.Context, opts ...GetOption) ([]*model.KVResponse, error) {
+//Search get value by labels
+func (c *Client) Search(ctx context.Context, opts ...GetOption) ([]*model.KVResponse, error) {
options := GetOptions{}
for _, o := range opts {
o(&options)
diff --git a/go.mod b/go.mod
index 9a2103c..39f57e5 100644
--- a/go.mod
+++ b/go.mod
@@ -6,7 +6,7 @@
github.com/go-chassis/go-archaius v0.24.0
github.com/go-chassis/go-chassis v1.7.6
github.com/go-chassis/go-chassis-config v0.15.0
- github.com/go-chassis/go-restful-swagger20 v1.0.2-0.20191118130439-7eec0f2639f6
+ github.com/go-chassis/go-restful-swagger20 v1.0.2-0.20191118130439-7eec0f2639f6 // indirect
github.com/go-chassis/paas-lager v1.0.2-0.20190328010332-cf506050ddb2
github.com/go-mesh/openlogging v1.0.1
github.com/golang/snappy v0.0.1 // indirect
diff --git a/pkg/model/kv.go b/pkg/model/kv.go
index 229ba7f..f603fe8 100644
--- a/pkg/model/kv.go
+++ b/pkg/model/kv.go
@@ -28,7 +28,10 @@
//KVResponse represents the key value list
type KVResponse struct {
- LabelDoc *LabelDocResponse `json:"label"`
+ LabelDoc *LabelDocResponse `json:"label,omitempty"`
+ PageNum int `json:"num,omitempty"`
+ Size int `json:"size,omitempty"`
+ Total int `json:"total,omitempty"`
Data []*KVDoc `json:"data"`
}
diff --git a/server/resource/v1/doc_struct.go b/server/resource/v1/doc_struct.go
index 2870376..b0afbb0 100644
--- a/server/resource/v1/doc_struct.go
+++ b/server/resource/v1/doc_struct.go
@@ -75,16 +75,19 @@
DataType: "string",
Name: "key",
ParamType: goRestful.PathParameterKind,
+ Required: true,
}
DocPathProject = &restful.Parameters{
DataType: "string",
Name: "project",
ParamType: goRestful.PathParameterKind,
+ Required: true,
}
DocPathLabelID = &restful.Parameters{
DataType: "string",
Name: "label_id",
ParamType: goRestful.PathParameterKind,
+ Required: true,
}
)
diff --git a/server/resource/v1/kv_resource.go b/server/resource/v1/kv_resource.go
index ee0db4a..b8e2010 100644
--- a/server/resource/v1/kv_resource.go
+++ b/server/resource/v1/kv_resource.go
@@ -20,6 +20,7 @@
import (
"net/http"
+ "strconv"
"github.com/apache/servicecomb-kie/pkg/common"
"github.com/apache/servicecomb-kie/pkg/model"
@@ -39,10 +40,6 @@
var err error
key := context.ReadPathParameter("key")
project := context.ReadPathParameter("project")
- if project == "" {
- WriteErrResponse(context, http.StatusForbidden, "project must not be empty", common.ContentTypeText)
- return
- }
kv := new(model.KVDoc)
if err = readRequest(context, kv); err != nil {
WriteErrResponse(context, http.StatusBadRequest, err.Error(), common.ContentTypeText)
@@ -78,10 +75,6 @@
return
}
project := context.ReadPathParameter("project")
- if project == "" {
- WriteErrResponse(context, http.StatusForbidden, "project must not be empty", common.ContentTypeText)
- return
- }
values := context.ReadRequest().URL.Query()
labels := make(map[string]string, len(values))
for k, v := range values {
@@ -118,8 +111,57 @@
}
-//SearchByLabels search key only by label
-func (r *KVResource) SearchByLabels(context *restful.Context) {
+//List TODO pagination
+func (r *KVResource) List(rctx *restful.Context) {
+ project := rctx.ReadPathParameter("project")
+ domain := ReadDomain(rctx)
+ if domain == nil {
+ WriteErrResponse(rctx, http.StatusInternalServerError, MsgDomainMustNotBeEmpty, common.ContentTypeText)
+ return
+ }
+ var limit int64 = 20
+ var offset int64 = 0
+ labels := make(map[string]string, 0)
+ var err error
+ for k, v := range rctx.ReadRequest().URL.Query() {
+ if k == "limit" {
+ limit, err = strconv.ParseInt(v[0], 10, 64)
+ if err != nil {
+ WriteErrResponse(rctx, http.StatusBadRequest, "invalid limit number", common.ContentTypeText)
+ }
+ if limit < 1 || limit > 50 {
+ WriteErrResponse(rctx, http.StatusBadRequest, "invalid limit number", common.ContentTypeText)
+ }
+ continue
+ }
+ if k == "offset" {
+ offset, err = strconv.ParseInt(v[0], 10, 64)
+ if err != nil {
+ WriteErrResponse(rctx, http.StatusBadRequest, "invalid offset number", common.ContentTypeText)
+ }
+ if offset < 1 {
+ WriteErrResponse(rctx, http.StatusBadRequest, "invalid offset number", common.ContentTypeText)
+ }
+ continue
+ }
+ labels[k] = v[0]
+ }
+ result, err := service.KVService.List(rctx.Ctx, domain.(string), project, "", labels, int(limit), int(offset))
+ if err != nil {
+ openlogging.Error("can not find by labels", openlogging.WithTags(openlogging.Tags{
+ "err": err.Error(),
+ }))
+ WriteErrResponse(rctx, http.StatusInternalServerError, err.Error(), common.ContentTypeText)
+ return
+ }
+ err = writeResponse(rctx, result)
+ if err != nil {
+ openlogging.Error(err.Error())
+ }
+}
+
+//Search search key only by label
+func (r *KVResource) Search(context *restful.Context) {
var err error
labelCombinations, err := ReadLabelCombinations(context.ReadRestfulRequest())
if err != nil {
@@ -127,16 +169,23 @@
return
}
project := context.ReadPathParameter("project")
- if project == "" {
- WriteErrResponse(context, http.StatusForbidden, "project must not be empty", common.ContentTypeText)
- return
- }
domain := ReadDomain(context)
if domain == nil {
WriteErrResponse(context, http.StatusInternalServerError, MsgDomainMustNotBeEmpty, common.ContentTypeText)
return
}
var kvs []*model.KVResponse
+ if labelCombinations == nil {
+ result, err := service.KVService.FindKV(context.Ctx, domain.(string), project)
+ if err != nil {
+ openlogging.Error("can not find by labels", openlogging.WithTags(openlogging.Tags{
+ "err": err.Error(),
+ }))
+ WriteErrResponse(context, http.StatusInternalServerError, err.Error(), common.ContentTypeText)
+ return
+ }
+ kvs = append(kvs, result...)
+ }
for _, labels := range labelCombinations {
openlogging.Debug("find by combination", openlogging.WithTags(openlogging.Tags{
"q": labels,
@@ -171,10 +220,6 @@
//Delete deletes key by ids
func (r *KVResource) Delete(context *restful.Context) {
project := context.ReadPathParameter("project")
- if project == "" {
- WriteErrResponse(context, http.StatusForbidden, "project must not be empty", common.ContentTypeText)
- return
- }
domain := ReadDomain(context)
if domain == nil {
WriteErrResponse(context, http.StatusInternalServerError, MsgDomainMustNotBeEmpty, common.ContentTypeText)
@@ -240,7 +285,7 @@
}, {
Method: http.MethodGet,
Path: "/v1/{project}/kie/kv",
- ResourceFunc: r.SearchByLabels,
+ ResourceFunc: r.Search,
FuncDesc: "search key values by labels combination",
Parameters: []*restful.Parameters{
DocPathProject, DocQueryCombination,
@@ -255,10 +300,26 @@
Consumes: []string{goRestful.MIME_JSON, common.ContentTypeYaml},
Produces: []string{goRestful.MIME_JSON, common.ContentTypeYaml},
}, {
+ Method: http.MethodGet,
+ Path: "/v1/{project}/kie/kv:list",
+ ResourceFunc: r.List,
+ FuncDesc: "list key values by labels and key",
+ Parameters: []*restful.Parameters{
+ DocPathProject, DocQueryLabelParameters,
+ },
+ Returns: []*restful.Returns{
+ {
+ Code: http.StatusOK,
+ Model: model.KVResponse{},
+ },
+ },
+ Consumes: []string{goRestful.MIME_JSON, common.ContentTypeYaml},
+ Produces: []string{goRestful.MIME_JSON, common.ContentTypeYaml},
+ }, {
Method: http.MethodDelete,
Path: "/v1/{project}/kie/kv",
ResourceFunc: r.Delete,
- FuncDesc: "delete key by kvID and labelID. if you want better performance, you need to give labelID",
+ FuncDesc: "delete key by kvID and labelID. Want better performance, give labelID",
Parameters: []*restful.Parameters{
DocPathProject,
DocQueryKVIDParameters,
diff --git a/server/resource/v1/kv_resource_test.go b/server/resource/v1/kv_resource_test.go
index c4c575e..5fe878b 100644
--- a/server/resource/v1/kv_resource_test.go
+++ b/server/resource/v1/kv_resource_test.go
@@ -22,6 +22,7 @@
"encoding/json"
"github.com/apache/servicecomb-kie/server/service"
"io/ioutil"
+ "log"
"net/http"
"net/http/httptest"
@@ -43,13 +44,12 @@
config.Configurations = &config.Config{
DB: config.DB{},
}
-
+ config.Configurations.DB.URI = "mongodb://kie:123@127.0.0.1:27017"
+ err := service.DBInit()
+ if err != nil {
+ panic(err)
+ }
Describe("put kv", func() {
- config.Configurations.DB.URI = "mongodb://kie:123@127.0.0.1:27017"
- err := service.DBInit()
- It("should not return err", func() {
- Expect(err).Should(BeNil())
- })
Context("valid param", func() {
kv := &model.KVDoc{
Key: "timeout",
@@ -86,4 +86,34 @@
})
})
})
+ Describe("list kv", func() {
+ Context("with no label", func() {
+ r, _ := http.NewRequest("GET", "/v1/test/kie/kv:list", nil)
+ noopH := &noop.NoopAuthHandler{}
+ chain, _ := handler.CreateChain(common.Provider, "testchain1", noopH.Name())
+ r.Header.Set("Content-Type", "application/json")
+ kvr := &v1.KVResource{}
+ c, err := restfultest.New(kvr, chain)
+ It("should not return error", func() {
+ Expect(err).Should(BeNil())
+ })
+ resp := httptest.NewRecorder()
+ c.ServeHTTP(resp, r)
+
+ body, err := ioutil.ReadAll(resp.Body)
+ It("should not return err", func() {
+ Expect(err).Should(BeNil())
+ })
+ log.Println(string(body))
+ result := &model.KVResponse{}
+ err = json.Unmarshal(body, result)
+ It("should not return err", func() {
+ Expect(err).Should(BeNil())
+ })
+
+ It("should longer than 1", func() {
+ Expect(len(result.Data)).NotTo(Equal(0))
+ })
+ })
+ })
})
diff --git a/server/service/mongo/kv/kv_service.go b/server/service/mongo/kv/kv_service.go
index e93eaa8..6b2e86e 100644
--- a/server/service/mongo/kv/kv_service.go
+++ b/server/service/mongo/kv/kv_service.go
@@ -192,6 +192,30 @@
return nil
}
+//List get kv list by key and criteria
+func (s *Service) List(ctx context.Context, domain, project, key string, labels map[string]string, limit, offset int) (*model.KVResponse, error) {
+ opts := service.NewDefaultFindOpts()
+ opts.Labels = labels
+ opts.Key = key
+ cur, err := findKV(ctx, domain, project, opts)
+ if err != nil {
+ return nil, err
+ }
+ defer cur.Close(ctx)
+ result := &model.KVResponse{}
+ for cur.Next(ctx) {
+ curKV := &model.KVDoc{}
+
+ if err := cur.Decode(curKV); err != nil {
+ openlogging.Error("decode to KVs error: " + err.Error())
+ return nil, err
+ }
+ clearPart(curKV)
+ result.Data = append(result.Data, curKV)
+ }
+ return result, nil
+}
+
//FindKV get kvs by key, labels
//because labels has a a lot of combination,
//you can use WithDepth(0) to return only one kv which's labels exactly match the criteria
@@ -251,7 +275,7 @@
for _, labelGroup = range kvResp {
if reflect.DeepEqual(labelGroup.LabelDoc.Labels, curKV.Labels) {
groupExist = true
- clearKV(curKV)
+ clearAll(curKV)
labelGroup.Data = append(labelGroup.Data, curKV)
break
}
@@ -265,7 +289,7 @@
},
Data: []*model.KVDoc{curKV},
}
- clearKV(curKV)
+ clearAll(curKV)
openlogging.Debug("add new label group")
kvResp = append(kvResp, labelGroup)
}
diff --git a/server/service/mongo/kv/tool.go b/server/service/mongo/kv/tool.go
index cbf40ec..c1a3f47 100644
--- a/server/service/mongo/kv/tool.go
+++ b/server/service/mongo/kv/tool.go
@@ -26,16 +26,18 @@
"go.mongodb.org/mongo-driver/mongo"
)
-//clearKV clean attr which don't need to return to client side
-func clearKV(kv *model.KVDoc) {
- kv.Domain = ""
+//clearAll clean attr which don't need to return to client side
+func clearAll(kv *model.KVDoc) {
+ clearPart(kv)
kv.Labels = nil
kv.LabelID = ""
+}
+func clearPart(kv *model.KVDoc) {
+ kv.Domain = ""
kv.Project = ""
}
-
func cursorToOneKV(ctx context.Context, cur *mongo.Cursor, labels map[string]string) ([]*model.KVResponse, error) {
- kvResp := make([]*model.KVResponse, 0)
+ result := make([]*model.KVResponse, 0)
curKV := &model.KVDoc{} //reuse this pointer to reduce GC, only clear label
//check label length to get the exact match
for cur.Next(ctx) { //although complexity is O(n), but there won't be so much labels for one key
@@ -57,10 +59,10 @@
},
Data: make([]*model.KVDoc, 0),
}
- clearKV(curKV)
+ clearAll(curKV)
labelGroup.Data = append(labelGroup.Data, curKV)
- kvResp = append(kvResp, labelGroup)
- return kvResp, nil
+ result = append(result, labelGroup)
+ return result, nil
}
}
diff --git a/server/service/options.go b/server/service/options.go
index e8eb5cb..9fe41c6 100644
--- a/server/service/options.go
+++ b/server/service/options.go
@@ -17,7 +17,17 @@
package service
-import "time"
+import (
+ "github.com/apache/servicecomb-kie/server/service/mongo/session"
+ "time"
+)
+
+//NewDefaultFindOpts return default options
+func NewDefaultFindOpts() FindOptions {
+ return FindOptions{
+ Timeout: session.DefaultTimeout,
+ }
+}
//FindOptions is option to find key value
type FindOptions struct {
diff --git a/server/service/service.go b/server/service/service.go
index 04310f6..5d067a2 100644
--- a/server/service/service.go
+++ b/server/service/service.go
@@ -38,8 +38,11 @@
//KV provide api of KV entity
type KV interface {
+ //below 3 methods is usually for admin console
CreateOrUpdate(ctx context.Context, kv *model.KVDoc) (*model.KVDoc, error)
+ List(ctx context.Context, domain, project, key string, labels map[string]string, limit, offset int) (*model.KVResponse, error)
Delete(kvID string, labelID string, domain, project string) error
+ //FindKV is usually for service to pull configs
FindKV(ctx context.Context, domain, project string, options ...FindOption) ([]*model.KVResponse, error)
}
