#79 record polling history (#94)
* record polling history
* fix bugs and use chan get body
* add unit test
* rename by comment
* use context instead of chan
* use const instead string
* modify as comment
diff --git a/deployments/db.js b/deployments/db.js
index 1fa2520..0c8acb2 100644
--- a/deployments/db.js
+++ b/deployments/db.js
@@ -96,11 +96,53 @@
}
} }
} );
+
+db.createCollection( "polling_detail", {
+ validator: { $jsonSchema: {
+ bsonType: "object",
+ required: [ "id","params","session_id","url_path" ],
+ properties: {
+ id: {
+ bsonType: "string",
+ },
+ session_id: {
+ bsonType: "string",
+ },
+ domain: {
+ bsonType: "string",
+ },
+ params: {
+ bsonType: "string"
+ },
+ ip: {
+ bsonType: "string"
+ },
+ user_agent: {
+ bsonType: "string"
+ },
+ url_path: {
+ bsonType: "string"
+ },
+ response_body: {
+ bsonType: "object"
+ },
+ response_header: {
+ bsonType: "object"
+ },
+ response_code: {
+ bsonType: "string"
+ }
+ }
+ } }
+} );
+
//index
db.kv.createIndex({"id": 1}, { unique: true } );
db.kv.createIndex({key: 1, label_id: 1,domain:1,project:1},{ unique: true });
db.label.createIndex({"id": 1}, { unique: true } );
db.label.createIndex({format: 1,domain:1,project:1},{ unique: true });
+db.polling_detail.createIndex({"id": 1}, { unique: true } );
+db.polling_detail.createIndex({session:1,domain:1}, { unique: true } );
db.view.createIndex({"id": 1}, { unique: true } );
db.view.createIndex({display:1,domain:1,project:1},{ unique: true });
//db config
diff --git a/pkg/common/common.go b/pkg/common/common.go
index a6bd6bb..4f2f34a 100644
--- a/pkg/common/common.go
+++ b/pkg/common/common.go
@@ -58,6 +58,7 @@
MsgInvalidWait = "wait param should be formed with number and time unit like 5s,100ms, and less than 5m"
MsgInvalidRev = "revision param should be formed with number greater than 0"
ErrKvIDMustNotEmpty = "must supply kv id if you want to remove key"
+ RespBodyContextKey = "responseBody"
MaxWait = 5 * time.Minute
)
diff --git a/pkg/iputil/ip_util.go b/pkg/iputil/ip_util.go
new file mode 100644
index 0000000..7e21854
--- /dev/null
+++ b/pkg/iputil/ip_util.go
@@ -0,0 +1,24 @@
+package iputil
+
+import (
+ "net"
+ "net/http"
+ "strings"
+)
+
+//ClientIP try to get ip from http header
+func ClientIP(r *http.Request) string {
+ xForwardedFor := r.Header.Get("X-Forwarded-For")
+ ip := strings.TrimSpace(strings.Split(xForwardedFor, ",")[0])
+ if ip != "" {
+ return ip
+ }
+ ip = strings.TrimSpace(r.Header.Get("X-Real-Ip"))
+ if ip != "" {
+ return ip
+ }
+ if ip, _, err := net.SplitHostPort(strings.TrimSpace(r.RemoteAddr)); err == nil {
+ return ip
+ }
+ return ""
+}
diff --git a/pkg/model/kv.go b/pkg/model/kv.go
index ac92854..d898329 100644
--- a/pkg/model/kv.go
+++ b/pkg/model/kv.go
@@ -52,3 +52,17 @@
Total int `json:"total,omitempty"`
Data []*ViewDoc `json:"data,omitempty"`
}
+
+//PollingDetail record operation history
+type PollingDetail struct {
+ ID string `json:"id,omitempty" yaml:"id,omitempty"`
+ SessionID string `json:"session_id,omitempty" yaml:"session_id,omitempty"`
+ Domain string `json:"domain,omitempty" yaml:"domain,omitempty"`
+ PollingData map[string]interface{} `json:"params,omitempty" yaml:"params,omitempty"`
+ IP string `json:"ip,omitempty" yaml:"ip,omitempty"`
+ UserAgent string `json:"user_agent,omitempty" yaml:"user_agent,omitempty"`
+ URLPath string `json:"url_path,omitempty" yaml:"url_path,omitempty"`
+ ResponseBody interface{} `json:"response_body,omitempty" yaml:"response_body,omitempty"`
+ ResponseHeader map[string][]string `json:"response_header,omitempty" yaml:"response_header,omitempty"`
+ ResponseCode int `json:"response_code,omitempty" yaml:"response_code,omitempty"`
+}
diff --git a/server/resource/v1/common.go b/server/resource/v1/common.go
index b99dc80..e121d9b 100644
--- a/server/resource/v1/common.go
+++ b/server/resource/v1/common.go
@@ -243,6 +243,7 @@
}
rctx.ReadResponseWriter().Header().Set(common.HeaderRevision, strconv.FormatInt(rev, 10))
err = writeResponse(rctx, kv)
+ rctx.Ctx = context.WithValue(rctx.Ctx, common.RespBodyContextKey, kv)
if err != nil {
openlogging.Error(err.Error())
}
diff --git a/server/resource/v1/kv_resource.go b/server/resource/v1/kv_resource.go
index 0054025..5db4912 100644
--- a/server/resource/v1/kv_resource.go
+++ b/server/resource/v1/kv_resource.go
@@ -21,12 +21,15 @@
import (
"fmt"
"github.com/apache/servicecomb-kie/pkg/common"
+ "github.com/apache/servicecomb-kie/pkg/iputil"
"github.com/apache/servicecomb-kie/pkg/model"
"github.com/apache/servicecomb-kie/server/pubsub"
"github.com/apache/servicecomb-kie/server/service"
+ "github.com/apache/servicecomb-kie/server/service/mongo/record"
goRestful "github.com/emicklei/go-restful"
"github.com/go-chassis/go-chassis/server/restful"
"github.com/go-mesh/openlogging"
+ uuid "github.com/satori/go.uuid"
"net/http"
)
@@ -108,13 +111,14 @@
WriteErrResponse(rctx, http.StatusBadRequest, err.Error(), common.ContentTypeText)
return
}
+ insID := rctx.ReadHeader("sessionID")
statusStr := rctx.ReadQueryParameter("status")
status, err := checkStatus(statusStr)
if err != nil {
WriteErrResponse(rctx, http.StatusBadRequest, err.Error(), common.ContentTypeText)
return
}
- returnData(rctx, domain, project, labels, pageNum, pageSize, status)
+ returnData(rctx, domain, project, labels, pageNum, pageSize, status, insID)
}
//List response kv list
@@ -138,18 +142,22 @@
WriteErrResponse(rctx, http.StatusBadRequest, err.Error(), common.ContentTypeText)
return
}
+ sessionID := rctx.ReadHeader("sessionID")
statusStr := rctx.ReadQueryParameter("status")
status, err := checkStatus(statusStr)
if err != nil {
WriteErrResponse(rctx, http.StatusBadRequest, err.Error(), common.ContentTypeText)
return
}
- returnData(rctx, domain, project, labels, pageNum, pageSize, status)
+ returnData(rctx, domain, project, labels, pageNum, pageSize, status, sessionID)
}
-func returnData(rctx *restful.Context, domain interface{}, project string, labels map[string]string, pageNum, pageSize int64, status string) {
+func returnData(rctx *restful.Context, domain interface{}, project string, labels map[string]string, pageNum, pageSize int64, status, sessionID string) {
revStr := rctx.ReadQueryParameter(common.QueryParamRev)
wait := rctx.ReadQueryParameter(common.QueryParamWait)
+ if sessionID != "" {
+ defer RecordPollingDetail(rctx, revStr, wait, domain.(string), project, labels, pageNum, pageSize, sessionID)
+ }
if revStr == "" {
if wait == "" {
queryAndResponse(rctx, domain, project, "", labels, pageNum, pageSize, status)
@@ -206,6 +214,35 @@
}
}
+//RecordPollingDetail to record data after get or list
+func RecordPollingDetail(context *restful.Context, revStr, wait, domain, project string, labels map[string]string, limit, offset int64, sessionID string) {
+ data := &model.PollingDetail{}
+ data.ID = uuid.NewV4().String()
+ data.SessionID = sessionID
+ data.Domain = domain
+ data.IP = iputil.ClientIP(context.Req.Request)
+ dataMap := map[string]interface{}{
+ "revStr": revStr,
+ "wait": wait,
+ "domain": domain,
+ "project": project,
+ "labels": labels,
+ "limit": limit,
+ "offset": offset,
+ }
+ data.PollingData = dataMap
+ data.UserAgent = context.Req.HeaderParameter("User-Agent")
+ data.URLPath = context.ReadRequest().Method + " " + context.ReadRequest().URL.Path
+ data.ResponseHeader = context.Resp.Header()
+ data.ResponseCode = context.Resp.StatusCode()
+ data.ResponseBody = context.Ctx.Value(common.RespBodyContextKey)
+ _, err := record.CreateOrUpdate(context.Ctx, data)
+ if err != nil {
+ openlogging.Warn("record polling detail failed" + err.Error())
+ return
+ }
+}
+
//Search search key only by label
func (r *KVResource) Search(context *restful.Context) {
var err error
diff --git a/server/resource/v1/kv_resource_test.go b/server/resource/v1/kv_resource_test.go
index b0370b2..93dea44 100644
--- a/server/resource/v1/kv_resource_test.go
+++ b/server/resource/v1/kv_resource_test.go
@@ -76,6 +76,7 @@
noopH := &handler2.NoopAuthHandler{}
chain, _ := handler.CreateChain(common.Provider, "testchain1", noopH.Name())
r.Header.Set("Content-Type", "application/json")
+ r.Header.Set("sessionID", "test")
kvr := &v1.KVResource{}
c, _ := restfultest.New(kvr, chain)
resp := httptest.NewRecorder()
@@ -101,6 +102,7 @@
noopH := &handler2.NoopAuthHandler{}
chain, _ := handler.CreateChain(common.Provider, "testchain1", noopH.Name())
r.Header.Set("Content-Type", "application/json")
+ r.Header.Set("sessionID", "test")
kvr := &v1.KVResource{}
c, _ := restfultest.New(kvr, chain)
resp := httptest.NewRecorder()
diff --git a/server/service/mongo/kv/kv_test.go b/server/service/mongo/kv/kv_test.go
index 69afb3a..b08c1c0 100644
--- a/server/service/mongo/kv/kv_test.go
+++ b/server/service/mongo/kv/kv_test.go
@@ -28,6 +28,7 @@
"testing"
)
+//
func TestService_CreateOrUpdate(t *testing.T) {
var err error
config.Configurations = &config.Config{DB: config.DB{URI: "mongodb://kie:123@127.0.0.1:27017/kie"}}
@@ -43,7 +44,7 @@
"service": "cart",
},
Domain: "default",
- Project: "test",
+ Project: "kv-test",
})
assert.NoError(t, err)
assert.NotEmpty(t, kv.ID)
@@ -58,9 +59,9 @@
"version": "1.0.0",
},
Domain: "default",
- Project: "test",
+ Project: "kv-test",
})
- oid, err := kvsvc.Exist(context.TODO(), "default", "timeout", "test", service.WithLabels(map[string]string{
+ oid, err := kvsvc.Exist(context.TODO(), "default", "timeout", "kv-test", service.WithLabels(map[string]string{
"app": "mall",
"service": "cart",
"version": "1.0.0",
@@ -78,10 +79,10 @@
"app": "mall",
},
Domain: "default",
- Project: "test",
+ Project: "kv-test",
})
assert.NoError(t, err)
- kvs1, err := kvsvc.FindKV(context.Background(), "default", "test",
+ kvs1, err := kvsvc.FindKV(context.Background(), "default", "kv-test",
service.WithKey("timeout"),
service.WithLabels(map[string]string{
"app": "mall",
@@ -95,15 +96,15 @@
"app": "mall",
},
Domain: "default",
- Project: "test",
+ Project: "kv-test",
})
assert.Equal(t, "3s", afterKV.Value)
- savedKV, err := kvsvc.Exist(context.Background(), "default", "timeout", "test", service.WithLabels(map[string]string{
+ savedKV, err := kvsvc.Exist(context.Background(), "default", "timeout", "kv-test", service.WithLabels(map[string]string{
"app": "mall",
}))
assert.NoError(t, err)
assert.Equal(t, afterKV.Value, savedKV.Value)
- kvs, err := kvsvc.FindKV(context.Background(), "default", "test",
+ kvs, err := kvsvc.FindKV(context.Background(), "default", "kv-test",
service.WithKey("timeout"),
service.WithLabels(map[string]string{
"app": "mall",
@@ -117,7 +118,7 @@
func TestService_FindKV(t *testing.T) {
kvsvc := &kv.Service{}
t.Run("exact find by kv and labels with label app", func(t *testing.T) {
- kvs, err := kvsvc.FindKV(context.Background(), "default", "test",
+ kvs, err := kvsvc.FindKV(context.Background(), "default", "kv-test",
service.WithKey("timeout"),
service.WithLabels(map[string]string{
"app": "mall",
@@ -127,7 +128,7 @@
assert.Equal(t, 1, len(kvs))
})
t.Run("greedy find by labels,with labels app ans service ", func(t *testing.T) {
- kvs, err := kvsvc.FindKV(context.Background(), "default", "test",
+ kvs, err := kvsvc.FindKV(context.Background(), "default", "kv-test",
service.WithLabels(map[string]string{
"app": "mall",
"service": "cart",
@@ -146,20 +147,20 @@
"env": "test",
},
Domain: "default",
- Project: "test",
+ Project: "kv-test",
})
assert.NoError(t, err)
- err = kvsvc.Delete(context.TODO(), kv1.ID, "default", "test")
+ err = kvsvc.Delete(context.TODO(), kv1.ID, "default", "kv-test")
assert.NoError(t, err)
})
t.Run("miss id", func(t *testing.T) {
- err := kvsvc.Delete(context.TODO(), "", "default", "test")
+ err := kvsvc.Delete(context.TODO(), "", "default", "kv-test")
assert.Error(t, err)
})
t.Run("miss domain", func(t *testing.T) {
- err := kvsvc.Delete(context.TODO(), "2", "", "test")
+ err := kvsvc.Delete(context.TODO(), "2", "", "kv-test")
assert.Equal(t, session.ErrMissingDomain, err)
})
}
diff --git a/server/service/mongo/record/polling_detail_dao.go b/server/service/mongo/record/polling_detail_dao.go
new file mode 100644
index 0000000..3e76bcb
--- /dev/null
+++ b/server/service/mongo/record/polling_detail_dao.go
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package record
+
+import (
+ "context"
+ "github.com/apache/servicecomb-kie/pkg/model"
+ "github.com/apache/servicecomb-kie/server/service/mongo/session"
+ "go.mongodb.org/mongo-driver/bson"
+ "go.mongodb.org/mongo-driver/mongo"
+)
+
+//CreateOrUpdate create a record or update exist record
+func CreateOrUpdate(ctx context.Context, detail *model.PollingDetail) (*model.PollingDetail, error) {
+ collection := session.GetDB().Collection(session.CollectionPollingDetail)
+ queryFilter := bson.M{"domain": detail.Domain, "session_id": detail.SessionID}
+ res := collection.FindOne(ctx, queryFilter)
+ if res.Err() != nil {
+ if res.Err() == mongo.ErrNoDocuments {
+ _, err := collection.InsertOne(ctx, detail)
+ if err != nil {
+ return nil, err
+ }
+ return detail, nil
+ }
+ return nil, res.Err()
+ }
+ _, err := collection.UpdateOne(ctx, queryFilter, detail)
+ if err != nil {
+ return nil, err
+ }
+ return detail, nil
+}
diff --git a/server/service/mongo/session/session.go b/server/service/mongo/session/session.go
index 246e164..40af57d 100644
--- a/server/service/mongo/session/session.go
+++ b/server/service/mongo/session/session.go
@@ -42,13 +42,14 @@
const (
DBName = "kie"
- CollectionLabel = "label"
- CollectionKV = "kv"
- CollectionKVRevision = "kv_revision"
- CollectionCounter = "counter"
- CollectionView = "view"
- DefaultTimeout = 5 * time.Second
- DefaultValueType = "text"
+ CollectionLabel = "label"
+ CollectionKV = "kv"
+ CollectionKVRevision = "kv_revision"
+ CollectionPollingDetail = "polling_detail"
+ CollectionCounter = "counter"
+ CollectionView = "view"
+ DefaultTimeout = 5 * time.Second
+ DefaultValueType = "text"
)
//db errors
