#79 add get interface for polling data (#98)
* add get interface for polling data
* change header
* db modify
* fix update failed
* modify as comment
* modify as comment
* go checker
* add handler to test
* modify
* when header is empty don't record
diff --git a/deployments/db.js b/deployments/db.js
index 0c8acb2..a3304df 100644
--- a/deployments/db.js
+++ b/deployments/db.js
@@ -98,21 +98,20 @@
} );
db.createCollection( "polling_detail", {
+ capped: true,
+ max: 100,
validator: { $jsonSchema: {
bsonType: "object",
- required: [ "id","params","session_id","url_path" ],
+ required: [ "id","session_id","domain","url_path" ],
properties: {
id: {
bsonType: "string",
},
- session_id: {
- bsonType: "string",
- },
domain: {
bsonType: "string",
},
params: {
- bsonType: "string"
+ bsonType: "object"
},
ip: {
bsonType: "string"
@@ -120,9 +119,6 @@
user_agent: {
bsonType: "string"
},
- url_path: {
- bsonType: "string"
- },
response_body: {
bsonType: "object"
},
@@ -130,7 +126,7 @@
bsonType: "object"
},
response_code: {
- bsonType: "string"
+ bsonType: "number"
}
}
} }
@@ -142,7 +138,7 @@
db.label.createIndex({"id": 1}, { unique: true } );
db.label.createIndex({format: 1,domain:1,project:1},{ unique: true });
db.polling_detail.createIndex({"id": 1}, { unique: true } );
-db.polling_detail.createIndex({session:1,domain:1}, { unique: true } );
+db.polling_detail.createIndex({session_id:1,domain:1}, { unique: true } );
db.view.createIndex({"id": 1}, { unique: true } );
db.view.createIndex({display:1,domain:1,project:1},{ unique: true });
//db config
diff --git a/pkg/common/common.go b/pkg/common/common.go
index 191d627..06f1118 100644
--- a/pkg/common/common.go
+++ b/pkg/common/common.go
@@ -31,6 +31,11 @@
QueryParamStatus = "status"
QueryParamOffset = "offset"
QueryParamLimit = "limit"
+ //polling data
+ QueryParamSessionID = "sessionId"
+ QueryParamIP = "ip"
+ QueryParamURLPath = "urlPath"
+ QueryParamUserAgent = "userAgent"
)
//http headers
diff --git a/pkg/model/db_schema.go b/pkg/model/db_schema.go
index 950e0cd..e786a85 100644
--- a/pkg/model/db_schema.go
+++ b/pkg/model/db_schema.go
@@ -55,3 +55,17 @@
Domain string `json:"domain,omitempty" yaml:"domain,omitempty"`
Criteria string `json:"criteria,omitempty" yaml:"criteria,omitempty"`
}
+
+//PollingDetail is db struct, it record operation history
+type PollingDetail struct {
+ ID string `json:"id,omitempty" yaml:"id,omitempty"`
+ SessionID string `json:"session_id,omitempty" bson:"session_id," yaml:"session_id,omitempty"`
+ Domain string `json:"domain,omitempty" yaml:"domain,omitempty"`
+ PollingData map[string]interface{} `json:"params,omitempty" yaml:"params,omitempty"`
+ IP string `json:"ip,omitempty" yaml:"ip,omitempty"`
+ UserAgent string `json:"user_agent,omitempty" bson:"user_agent," yaml:"user_agent,omitempty"`
+ URLPath string `json:"url_path,omitempty" bson:"url_path," yaml:"url_path,omitempty"`
+ ResponseBody interface{} `json:"response_body,omitempty" bson:"response_body," yaml:"response_body,omitempty"`
+ ResponseHeader map[string][]string `json:"response_header,omitempty" bson:"response_header," yaml:"response_header,omitempty"`
+ ResponseCode int `json:"response_code,omitempty" bson:"response_code," yaml:"response_code,omitempty"`
+}
diff --git a/pkg/model/kv.go b/pkg/model/kv.go
index b002037..71d20cd 100644
--- a/pkg/model/kv.go
+++ b/pkg/model/kv.go
@@ -53,20 +53,6 @@
Data []*ViewDoc `json:"data,omitempty"`
}
-//PollingDetail record operation history
-type PollingDetail struct {
- ID string `json:"id,omitempty" yaml:"id,omitempty"`
- SessionID string `json:"session_id,omitempty" yaml:"session_id,omitempty"`
- Domain string `json:"domain,omitempty" yaml:"domain,omitempty"`
- PollingData map[string]interface{} `json:"params,omitempty" yaml:"params,omitempty"`
- IP string `json:"ip,omitempty" yaml:"ip,omitempty"`
- UserAgent string `json:"user_agent,omitempty" yaml:"user_agent,omitempty"`
- URLPath string `json:"url_path,omitempty" yaml:"url_path,omitempty"`
- ResponseBody interface{} `json:"response_body,omitempty" yaml:"response_body,omitempty"`
- ResponseHeader map[string][]string `json:"response_header,omitempty" yaml:"response_header,omitempty"`
- ResponseCode int `json:"response_code,omitempty" yaml:"response_code,omitempty"`
-}
-
//DocResponseSingleKey is response doc
type DocResponseSingleKey struct {
CreateRevision int64 `json:"create_revision"`
@@ -87,6 +73,12 @@
Total int64 `json:"total"`
}
+//PollingDataResponse is response doc
+type PollingDataResponse struct {
+ Data []*PollingDetail `json:"data"`
+ Total int `json:"total"`
+}
+
//DocHealthCheck is response doc
type DocHealthCheck struct {
Version string `json:"version"`
diff --git a/server/handler/track_handler.go b/server/handler/track_handler.go
index f85df93..3d6979d 100644
--- a/server/handler/track_handler.go
+++ b/server/handler/track_handler.go
@@ -22,7 +22,7 @@
"github.com/apache/servicecomb-kie/pkg/iputil"
"github.com/apache/servicecomb-kie/pkg/model"
"github.com/apache/servicecomb-kie/server/resource/v1"
- "github.com/apache/servicecomb-kie/server/service/mongo/record"
+ "github.com/apache/servicecomb-kie/server/service/mongo/track"
"github.com/emicklei/go-restful"
"github.com/go-chassis/go-chassis/core/handler"
"github.com/go-chassis/go-chassis/core/invocation"
@@ -60,14 +60,7 @@
return
}
chain.Next(inv, func(ir *invocation.Response) error {
- resp, ok := ir.Result.(*restful.Response)
- if !ok {
- err := cb(ir)
- if err != nil {
- return err
- }
- return nil
- }
+ resp, _ := ir.Result.(*restful.Response)
revStr := req.QueryParameter(common.QueryParamRev)
wait := req.QueryParameter(common.QueryParamWait)
data := &model.PollingDetail{}
@@ -78,14 +71,16 @@
data.IP = iputil.ClientIP(req.Request)
data.ResponseBody = inv.Ctx.Value(common.RespBodyContextKey)
data.ResponseCode = ir.Status
- data.ResponseHeader = resp.Header()
+ if resp != nil {
+ data.ResponseHeader = resp.Header()
+ }
data.PollingData = map[string]interface{}{
"revStr": revStr,
"wait": wait,
"project": req.HeaderParameter(v1.PathParameterProject),
"labels": req.QueryParameter("label"),
}
- _, err := record.CreateOrUpdate(inv.Ctx, data)
+ _, err := track.CreateOrUpdate(inv.Ctx, data)
if err != nil {
openlogging.Warn("record polling detail failed" + err.Error())
err := cb(ir)
diff --git a/server/resource/v1/common.go b/server/resource/v1/common.go
index 957308f..2d48b79 100644
--- a/server/resource/v1/common.go
+++ b/server/resource/v1/common.go
@@ -40,7 +40,7 @@
//const of server
const (
HeaderUserAgent = "User-Agent"
- HeaderSessionID = "sessionID"
+ HeaderSessionID = "X-Session-Id"
PathParameterProject = "project"
PathParameterKey = "key"
AttributeDomainKey = "domain"
diff --git a/server/resource/v1/doc_struct.go b/server/resource/v1/doc_struct.go
index 5c3e35c..1a5aebb 100644
--- a/server/resource/v1/doc_struct.go
+++ b/server/resource/v1/doc_struct.go
@@ -101,6 +101,31 @@
ParamType: goRestful.QueryParameterKind,
Desc: "pagination",
}
+ //polling data
+ DocQuerySessionIDParameters = &restful.Parameters{
+ DataType: "string",
+ Name: common.QueryParamSessionID,
+ ParamType: goRestful.QueryParameterKind,
+ Desc: "sessionId is the Unique identification of the client",
+ }
+ DocQueryIPParameters = &restful.Parameters{
+ DataType: "string",
+ Name: common.QueryParamIP,
+ ParamType: goRestful.QueryParameterKind,
+ Desc: "client ip",
+ }
+ DocQueryURLPathParameters = &restful.Parameters{
+ DataType: "string",
+ Name: common.QueryParamURLPath,
+ ParamType: goRestful.QueryParameterKind,
+ Desc: "address of the call",
+ }
+ DocQueryUserAgentParameters = &restful.Parameters{
+ DataType: "string",
+ Name: common.QueryParamUserAgent,
+ ParamType: goRestful.QueryParameterKind,
+ Desc: "user agent of the call",
+ }
)
//swagger doc path params
diff --git a/server/resource/v1/history_resource.go b/server/resource/v1/history_resource.go
index 32376c1..9b6269a 100644
--- a/server/resource/v1/history_resource.go
+++ b/server/resource/v1/history_resource.go
@@ -19,6 +19,7 @@
import (
"github.com/apache/servicecomb-kie/pkg/model"
+ "github.com/apache/servicecomb-kie/server/service/mongo/track"
"github.com/go-chassis/go-chassis/pkg/runtime"
"net/http"
"strconv"
@@ -75,6 +76,49 @@
}
}
+//GetPollingData get the record of the get or list history
+func (r *HistoryResource) GetPollingData(context *restful.Context) {
+ query := &model.PollingDetail{}
+ sessionID := context.ReadQueryParameter(common.QueryParamSessionID)
+ if sessionID != "" {
+ query.SessionID = sessionID
+ }
+ ip := context.ReadQueryParameter(common.QueryParamIP)
+ if ip != "" {
+ query.IP = ip
+ }
+ urlPath := context.ReadQueryParameter(common.QueryParamURLPath)
+ if urlPath != "" {
+ query.URLPath = urlPath
+ }
+ userAgent := context.ReadQueryParameter(common.QueryParamUserAgent)
+ if userAgent != "" {
+ query.UserAgent = userAgent
+ }
+ domain := ReadDomain(context)
+ if domain == nil {
+ WriteErrResponse(context, http.StatusInternalServerError, common.MsgDomainMustNotBeEmpty, common.ContentTypeText)
+ return
+ }
+ query.Domain = domain.(string)
+ records, err := track.Get(context.Ctx, query)
+ if err != nil {
+ if err == service.ErrRecordNotExists {
+ WriteErrResponse(context, http.StatusNotFound, err.Error(), common.ContentTypeText)
+ return
+ }
+ WriteErrResponse(context, http.StatusInternalServerError, err.Error(), common.ContentTypeText)
+ return
+ }
+ resp := &model.PollingDataResponse{}
+ resp.Data = records
+ resp.Total = len(records)
+ err = writeResponse(context, resp)
+ if err != nil {
+ openlogging.Error(err.Error())
+ }
+}
+
//HealthCheck provider version info and time info
func (r *HistoryResource) HealthCheck(context *restful.Context) {
domain := ReadDomain(context)
@@ -128,5 +172,23 @@
Consumes: []string{goRestful.MIME_JSON, common.ContentTypeYaml},
Produces: []string{goRestful.MIME_JSON, common.ContentTypeYaml},
},
+ {
+ Method: http.MethodGet,
+ Path: "/v1/{project}/kie/track",
+ ResourceFunc: r.GetPollingData,
+ FuncDesc: "get polling tracks of clients of kie server",
+ Parameters: []*restful.Parameters{
+ DocPathProject, DocQuerySessionIDParameters, DocQueryIPParameters, DocQueryURLPathParameters, DocQueryUserAgentParameters,
+ },
+ Returns: []*restful.Returns{
+ {
+ Code: http.StatusOK,
+ Message: "true",
+ Model: []model.PollingDataResponse{},
+ },
+ },
+ Consumes: []string{goRestful.MIME_JSON, common.ContentTypeYaml},
+ Produces: []string{goRestful.MIME_JSON, common.ContentTypeYaml},
+ },
}
}
diff --git a/server/resource/v1/history_resource_test.go b/server/resource/v1/history_resource_test.go
index 2a05a56..e7f9ed8 100644
--- a/server/resource/v1/history_resource_test.go
+++ b/server/resource/v1/history_resource_test.go
@@ -85,6 +85,45 @@
}
+func TestHistoryResource_GetPollingData(t *testing.T) {
+ t.Run("list kv by service label, to create a polling data", func(t *testing.T) {
+ r, _ := http.NewRequest("GET", "/v1/test/kie/kv", nil)
+ noopH := &handler2.NoopAuthHandler{}
+ noopH2 := &handler2.TrackHandler{}
+ chain, _ := handler.CreateChain(common.Provider, "testchain3", noopH.Name(), noopH2.Name())
+ r.Header.Set("Content-Type", "application/json")
+ r.Header.Set("X-Session-Id", "test")
+ kvr := &v1.KVResource{}
+ c, err := restfultest.New(kvr, chain)
+ assert.NoError(t, err)
+ resp := httptest.NewRecorder()
+ c.ServeHTTP(resp, r)
+ body, err := ioutil.ReadAll(resp.Body)
+ assert.NoError(t, err)
+ result := &model.KVResponse{}
+ err = json.Unmarshal(body, result)
+ assert.NoError(t, err)
+ })
+ t.Run("get polling data", func(t *testing.T) {
+ r, _ := http.NewRequest("GET", "/v1/test/kie/track?sessionId=test", nil)
+ noopH := &handler2.NoopAuthHandler{}
+ chain, _ := handler.CreateChain(common.Provider, "testchain3", noopH.Name())
+ r.Header.Set("Content-Type", "application/json")
+ revision := &v1.HistoryResource{}
+ c, err := restfultest.New(revision, chain)
+ assert.NoError(t, err)
+ resp := httptest.NewRecorder()
+ c.ServeHTTP(resp, r)
+ body, err := ioutil.ReadAll(resp.Body)
+ assert.NoError(t, err)
+ result := &model.PollingDataResponse{}
+ err = json.Unmarshal(body, result)
+ assert.NoError(t, err)
+ assert.NotEmpty(t, result.Data)
+ })
+
+}
+
func Test_HeathCheck(t *testing.T) {
path := fmt.Sprintf("/v1/health")
r, _ := http.NewRequest("GET", path, nil)
diff --git a/server/resource/v1/kv_resource_test.go b/server/resource/v1/kv_resource_test.go
index 35aa46f..e0450d7 100644
--- a/server/resource/v1/kv_resource_test.go
+++ b/server/resource/v1/kv_resource_test.go
@@ -76,7 +76,6 @@
noopH := &handler2.NoopAuthHandler{}
chain, _ := handler.CreateChain(common.Provider, "testchain1", noopH.Name())
r.Header.Set("Content-Type", "application/json")
- r.Header.Set("sessionID", "test")
kvr := &v1.KVResource{}
c, _ := restfultest.New(kvr, chain)
resp := httptest.NewRecorder()
@@ -101,7 +100,6 @@
noopH := &handler2.NoopAuthHandler{}
chain, _ := handler.CreateChain(common.Provider, "testchain1", noopH.Name())
r.Header.Set("Content-Type", "application/json")
- r.Header.Set("sessionID", "test")
kvr := &v1.KVResource{}
c, _ := restfultest.New(kvr, chain)
resp := httptest.NewRecorder()
@@ -127,7 +125,6 @@
noopH := &handler2.NoopAuthHandler{}
chain, _ := handler.CreateChain(common.Provider, "testchain1", noopH.Name())
r.Header.Set("Content-Type", "application/json")
- r.Header.Set("sessionID", "test")
kvr := &v1.KVResource{}
c, _ := restfultest.New(kvr, chain)
resp := httptest.NewRecorder()
diff --git a/server/service/mongo/record/polling_detail_dao.go b/server/service/mongo/record/polling_detail_dao.go
deleted file mode 100644
index 6cefe5c..0000000
--- a/server/service/mongo/record/polling_detail_dao.go
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package record
-
-import (
- "context"
- "github.com/apache/servicecomb-kie/pkg/model"
- "github.com/apache/servicecomb-kie/server/service/mongo/session"
- uuid "github.com/satori/go.uuid"
- "go.mongodb.org/mongo-driver/bson"
- "go.mongodb.org/mongo-driver/mongo"
-)
-
-//CreateOrUpdate create a record or update exist record
-func CreateOrUpdate(ctx context.Context, detail *model.PollingDetail) (*model.PollingDetail, error) {
- collection := session.GetDB().Collection(session.CollectionPollingDetail)
- queryFilter := bson.M{"domain": detail.Domain, "session_id": detail.SessionID}
- res := collection.FindOne(ctx, queryFilter)
- if res.Err() != nil {
- if res.Err() == mongo.ErrNoDocuments {
- detail.ID = uuid.NewV4().String()
- _, err := collection.InsertOne(ctx, detail)
- if err != nil {
- return nil, err
- }
- return detail, nil
- }
- return nil, res.Err()
- }
- _, err := collection.UpdateOne(ctx, queryFilter, detail)
- if err != nil {
- return nil, err
- }
- return detail, nil
-}
diff --git a/server/service/mongo/track/polling_detail_dao.go b/server/service/mongo/track/polling_detail_dao.go
new file mode 100644
index 0000000..998931d
--- /dev/null
+++ b/server/service/mongo/track/polling_detail_dao.go
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package track
+
+import (
+ "context"
+ "github.com/apache/servicecomb-kie/pkg/model"
+ "github.com/apache/servicecomb-kie/server/service"
+ "github.com/apache/servicecomb-kie/server/service/mongo/session"
+ "github.com/go-mesh/openlogging"
+ uuid "github.com/satori/go.uuid"
+ "go.mongodb.org/mongo-driver/bson"
+ "go.mongodb.org/mongo-driver/mongo"
+)
+
+//CreateOrUpdate create a record or update exist record
+func CreateOrUpdate(ctx context.Context, detail *model.PollingDetail) (*model.PollingDetail, error) {
+ collection := session.GetDB().Collection(session.CollectionPollingDetail)
+ queryFilter := bson.M{"domain": detail.Domain, "session_id": detail.SessionID}
+ res := collection.FindOne(ctx, queryFilter)
+ if res.Err() != nil {
+ if res.Err() == mongo.ErrNoDocuments {
+ detail.ID = uuid.NewV4().String()
+ _, err := collection.InsertOne(ctx, detail)
+ if err != nil {
+ return nil, err
+ }
+ return detail, nil
+ }
+ return nil, res.Err()
+ }
+ _, err := collection.UpdateOne(ctx, queryFilter, bson.D{{"$set", detail}})
+ if err != nil {
+ return nil, err
+ }
+ return detail, nil
+}
+
+//Get is to get a
+func Get(ctx context.Context, detail *model.PollingDetail) ([]*model.PollingDetail, error) {
+ collection := session.GetDB().Collection(session.CollectionPollingDetail)
+ queryFilter := bson.M{"domain": detail.Domain}
+ if detail.SessionID != "" {
+ queryFilter["session_id"] = detail.SessionID
+ }
+ if detail.IP != "" {
+ queryFilter["ip"] = detail.IP
+ }
+ if detail.UserAgent != "" {
+ queryFilter["user_agent"] = detail.UserAgent
+ }
+ if detail.URLPath != "" {
+ queryFilter["url_path"] = detail.URLPath
+ }
+ cur, err := collection.Find(ctx, queryFilter)
+ if err != nil {
+ return nil, err
+ }
+ defer cur.Close(ctx)
+ if cur.Err() != nil {
+ return nil, err
+ }
+ records := make([]*model.PollingDetail, 0)
+ for cur.Next(ctx) {
+ curRecord := &model.PollingDetail{}
+ if err := cur.Decode(curRecord); err != nil {
+ openlogging.Error("decode to KVs error: " + err.Error())
+ return nil, err
+ }
+ curRecord.Domain = ""
+ records = append(records, curRecord)
+ }
+ if len(records) == 0 {
+ return nil, service.ErrRecordNotExists
+ }
+ return records, nil
+}
diff --git a/server/service/service.go b/server/service/service.go
index c1451d5..5720a6c 100644
--- a/server/service/service.go
+++ b/server/service/service.go
@@ -35,6 +35,7 @@
//db errors
var (
ErrKeyNotExists = errors.New("can not find any key value")
+ ErrRecordNotExists = errors.New("can not find any polling data")
ErrRevisionNotExist = errors.New("revision does not exist")
ErrAliasNotGiven = errors.New("label alias not given")
)
