+ added sample logging mechanisms for StatsD in api-gateway-config, which for now it's only activated when the container is deployed in Marathon
diff --git a/Dockerfile b/Dockerfile
index 9a160fa..0ae5ea4 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -8,7 +8,7 @@
RUN apk update \
&& apk add gcc tar libtool zlib jemalloc jemalloc-dev perl \
- make musl-dev openssl-dev pcre-dev g++ zlib-dev curl
+ make musl-dev openssl-dev pcre-dev g++ zlib-dev curl python
ENV OPENRESTY_VERSION 1.9.3.1
ENV NAXSI_VERSION 0.53-2
diff --git a/api-gateway-config/api-gateway-reload-config.sh b/api-gateway-config/api-gateway-reload-config.sh
deleted file mode 100755
index bbe7a5a..0000000
--- a/api-gateway-config/api-gateway-reload-config.sh
+++ /dev/null
@@ -1,63 +0,0 @@
-#/*
-# * Copyright (c) 2012 Adobe Systems Incorporated. All rights reserved.
-# *
-# * Permission is hereby granted, free of charge, to any person obtaining a
-# * copy of this software and associated documentation files (the "Software"),
-# * to deal in the Software without restriction, including without limitation
-# * the rights to use, copy, modify, merge, publish, distribute, sublicense,
-# * and/or sell copies of the Software, and to permit persons to whom the
-# * Software is furnished to do so, subject to the following conditions:
-# *
-# * The above copyright notice and this permission notice shall be included in
-# * all copies or substantial portions of the Software.
-# *
-# * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-# * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-# * DEALINGS IN THE SOFTWARE.
-# *
-# */
-#!/bin/sh
-
-# this script checks to see if there's any modification to the configuration files since the last run
-# to automatically reload the configuration.
-
-# NOTE: not used for now
-
-# The script may be executed as part of a cronjob running every minute like
-# * * * * * /etc/api-gateway/api-gateway-reload-config.sh 1>&2 > /var/log/api-gateway/reload-config-cron.log
-
-LOG_DIR=/var/log/api-gateway
-LOG_FILE=$LOG_DIR/reload-config.log
-
-function do_log()
-{
- local _MSG=$1
- echo "[`date +'%Y-%m-%d-%H:%M:%S'`] - ${_MSG}"
-}
-
-function fatal_error()
-{
- local _MSG=$1
- do_log "ERROR: ${_MSG}"
- exit 255
-}
-
-function info_log()
-{
- local _MSG=$1
- do_log "${_MSG}"
-}
-
-
-changed_files=$(find /etc/api-gateway -type f -newer /var/run/apigateway-config-watcher.lastrun -print)
-if [[ -n "${changed_files}" ]]; then
- info_log "discovered changed files ..."
- info_log ${changed_files}
- info_log "reloading gateway ..."
- api-gateway -t -p /usr/local/api-gateway/ -c /etc/api-gateway/api-gateway.conf && api-gateway -s reload
-fi
-echo `date` > /var/run/apigateway-config-watcher.lastrun
\ No newline at end of file
diff --git a/api-gateway-config/conf.d/api_gateway_init.conf b/api-gateway-config/conf.d/api_gateway_init.conf
index e21699e..1c206e5 100644
--- a/api-gateway-config/conf.d/api_gateway_init.conf
+++ b/api-gateway-config/conf.d/api_gateway_init.conf
@@ -47,10 +47,15 @@
# be prepared to load any custom lua scripts from /etc/api-gateway/scripts
-lua_package_path '/etc/api-gateway/scripts/?.lua;;';
-init_worker_by_lua_file /etc/api-gateway/scripts/api_gateway_init.lua;
+lua_package_path '/etc/api-gateway/scripts/lua/?.lua;;';
+init_worker_by_lua_file /etc/api-gateway/scripts/lua/api_gateway_init.lua;
lua_shared_dict cachedkeys 50m; # caches api-keys
lua_shared_dict cachedOauthTokens 50m; # caches OAuth tokens
lua_shared_dict cachedUserProfiles 50m; # caches user profiles
-lua_shared_dict healthcheck_redis 1m; # used by lua health_check for redis cache
\ No newline at end of file
+lua_shared_dict healthcheck_redis 1m; # used by lua health_check for redis cache
+
+# metrics
+lua_shared_dict stats_counters 50m;
+lua_shared_dict stats_timers 50m;
+lua_shared_dict stats_all 50m;
\ No newline at end of file
diff --git a/api-gateway-config/conf.d/includes/analytics_endpoints.conf b/api-gateway-config/conf.d/includes/analytics_endpoints.conf
index 2c4ea0c..3dbca5e 100644
--- a/api-gateway-config/conf.d/includes/analytics_endpoints.conf
+++ b/api-gateway-config/conf.d/includes/analytics_endpoints.conf
@@ -27,7 +27,7 @@
allow 127.0.0.1;
deny all;
content_by_lua '
- local MetricsCls = require "legacy.metrics"
+ local MetricsCls = require "metrics.MetricsBuffer"
local metrics = MetricsCls:new()
local json = assert( metrics:toJson(), "Could not read metrics")
ngx.say( json )
diff --git a/api-gateway-config/marathon-service-discovery.sh b/api-gateway-config/marathon-service-discovery.sh
index 03caa2e..dd0d265 100755
--- a/api-gateway-config/marathon-service-discovery.sh
+++ b/api-gateway-config/marathon-service-discovery.sh
@@ -1,3 +1,4 @@
+#!/bin/sh
#/*
# * Copyright (c) 2012 Adobe Systems Incorporated. All rights reserved.
# *
@@ -20,7 +21,6 @@
# * DEALINGS IN THE SOFTWARE.
# *
# */
-#!/bin/sh
#
# Overview:
@@ -54,6 +54,9 @@
# NOTE: for the moment when tasks expose multiple ports, only the first one is exposed through nginx
curl -s ${marathon_host}/v2/tasks -H "Accept:text/plain" | awk 'NF>2' | grep -v :0 | awk '!seen[$1]++' | awk ' {s=""; for (f=3; f<=NF; f++) s = s "\n server " $f " fail_timeout=10s;" ; print "upstream " $1 " {" s "\n keepalive 16;\n}" }' > ${TMP_FILE}
# 1.1. check redis upstreams
+#
+# ASSUMPTION: there is a redis app named "api-gateway-redis" deployed in marathon and optionally another app named "api-gateway-redis-replica"
+#
redis_master=$(cat ${TMP_FILE} | grep api-gateway-redis | wc -l)
redis_replica=$(cat ${TMP_FILE} | grep api-gateway-redis-replica | wc -l)
# if api-gateway-redis upstream exists but api-gateway-redis-replica does not, then create the replica
diff --git a/api-gateway-config/scripts/api_gateway_init.lua b/api-gateway-config/scripts/lua/api_gateway_init.lua
similarity index 93%
rename from api-gateway-config/scripts/api_gateway_init.lua
rename to api-gateway-config/scripts/lua/api_gateway_init.lua
index b511058..db5a42c 100644
--- a/api-gateway-config/scripts/api_gateway_init.lua
+++ b/api-gateway-config/scripts/lua/api_gateway_init.lua
@@ -48,7 +48,12 @@
parentObject.validation = require "api-gateway.validation.factory"
end
+local function initMetricsFactory(parentObject)
+ parentObject.metrics = require "metrics.factory"
+end
+
initValidationFactory(_M)
+initMetricsFactory(_M)
-- TODO: test health-check with the new version of Openresty
-- initRedisHealthCheck()
diff --git a/api-gateway-config/scripts/lua/metrics/MetricsBuffer.lua b/api-gateway-config/scripts/lua/metrics/MetricsBuffer.lua
new file mode 100644
index 0000000..47d322b
--- /dev/null
+++ b/api-gateway-config/scripts/lua/metrics/MetricsBuffer.lua
@@ -0,0 +1,165 @@
+-- Copyright (c) 2015 Adobe Systems Incorporated. All rights reserved.
+--
+-- Permission is hereby granted, free of charge, to any person obtaining a
+-- copy of this software and associated documentation files (the "Software"),
+-- to deal in the Software without restriction, including without limitation
+-- the rights to use, copy, modify, merge, publish, distribute, sublicense,
+-- and/or sell copies of the Software, and to permit persons to whom the
+-- Software is furnished to do so, subject to the following conditions:
+--
+-- The above copyright notice and this permission notice shall be included in
+-- all copies or substantial portions of the Software.
+--
+-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+-- FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+-- DEALINGS IN THE SOFTWARE.
+
+-- Base class for analytics with functions for calculating counts,timers
+-- and updating the cache with data on every service call.
+--
+--
+--
+-- Usage in Nginx conf:
+--
+-- location api-gateway-stats {
+-- allow 127.0.0.1;
+-- deny all;
+-- content_by_lua '
+-- local MetricsBuffer = require "metrics.MetricsBuffer"
+-- local metrics = MetricsBuffer:new()
+-- local json = assert( metrics:toJson(), "Could not read metrics")
+-- ngx.say( json )
+-- ';
+-- }
+
+local cjson = require "cjson"
+
+local M = {}
+
+function M:new(o)
+ o = o or {}
+ setmetatable(o, self)
+ self.__index = self
+ return o
+end
+
+--- Increments a counter with the given <value>
+-- @param metric The name of the metrics
+-- @param value The value to intecrement with
+--
+function M:count(metric, value)
+ local localMetrics = ngx.shared.stats_counters
+ local r
+ if ( localMetrics == nil ) then
+ ngx.log(ngx.ERROR, "Please define 'lua_shared_dict stats_counters 50m;' in http block")
+ return nil
+ end
+
+ value = tonumber(value) or 0
+ if value == 0 then -- to exit when nil/zero
+ return 0
+ end
+
+ r = localMetrics:incr(metric, value)
+ if ( r == nil ) then
+ localMetrics:add(metric, value)
+ return value
+ end
+ return r
+end
+
+--- Adds a new timer value. Unline counters, timers are averaged when flushed
+-- @param metric The name of the metrics
+-- @param value A new value for the timer
+--
+function M:timer(metric, value)
+ local localMetrics = ngx.shared.stats_timers
+ local r, counter
+ if ( localMetrics == nil ) then
+ ngx.log(ngx.ERROR, "Please define 'lua_shared_dict stats_timers 50m;' in http block")
+ return nil
+ end
+
+ value = tonumber(value) or -2
+
+ -- Timers are counting only the positive values, by convention
+ if ( value < 0 ) then
+ return value
+ end
+
+ r, counter = localMetrics:get(metric)
+ if ( r == nil ) then
+ localMetrics:set(metric, value, 0, 1)
+ return value
+ end
+ -- FIXES: attempt to perform arithmetic on local 'counter' (a nil value)
+ if ( counter == nil ) then
+ counter = 0
+ end
+ -- Adding the timers and increamenting the counter to compute avg later
+ counter = counter + 1
+ r = r + value
+ localMetrics:set(metric, r, 0, counter)
+ return r
+end
+
+function M:getJsonFor( metric_type )
+ -- convert shared_dict to table
+ local localMetrics = ngx.shared[metric_type]
+ if ( localMetrics == nil ) then
+ return nil
+ end
+ local keys = localMetrics:get_keys(1024)
+ local counter
+ local jsonObj = {}
+ local count = 0
+ for i,metric in pairs(keys) do
+ local value, counter = localMetrics:get(metric)
+ if(counter == nil) then
+ counter = 0
+ end
+ -- check if avg needs to be computed
+ if(counter == 1 or counter == 0) then
+ jsonObj[metric] = value
+ end
+ if(counter >= 2) then
+ jsonObj[metric] = math.floor(value/counter)
+ end
+ -- mark item as expired
+ localMetrics:set(metric, 0, 0.001,0)
+ count = i
+ end
+ return cjson.encode(jsonObj),count,jsonObj
+end
+
+function M:toJson( flushExpiredMetrics )
+ local counters, count_counters, counterObject = self:getJsonFor("stats_counters")
+ local timers, count_timers, timerObject = self:getJsonFor("stats_timers")
+ local flush = flushExpiredMetrics or true
+
+ -- ngx.log(ngx.INFO, "Wrote " .. count_counters .. " counters and " .. count_timers .. " timers.")
+
+ if ( flush == true ) then
+ self:flushExpiredKeys()
+ end
+ return "{\"counters\":" .. counters .. ",\"timers\":" .. timers .. "}", counterObject, timerObject
+end
+
+function M:flushExpiredKeys()
+ local metrics = ngx.shared.stats_counters
+ if ( metrics ~= nil ) then
+ metrics:flush_expired()
+ end
+
+ metrics = ngx.shared.stats_timers
+ if ( metrics ~= nil ) then
+ metrics:flush_expired()
+ end
+end
+
+return M
+
diff --git a/api-gateway-config/scripts/lua/metrics/MetricsCollector.lua b/api-gateway-config/scripts/lua/metrics/MetricsCollector.lua
new file mode 100644
index 0000000..624fb1f
--- /dev/null
+++ b/api-gateway-config/scripts/lua/metrics/MetricsCollector.lua
@@ -0,0 +1,185 @@
+-- Copyright (c) 2015 Adobe Systems Incorporated. All rights reserved.
+--
+-- Permission is hereby granted, free of charge, to any person obtaining a
+-- copy of this software and associated documentation files (the "Software"),
+-- to deal in the Software without restriction, including without limitation
+-- the rights to use, copy, modify, merge, publish, distribute, sublicense,
+-- and/or sell copies of the Software, and to permit persons to whom the
+-- Software is furnished to do so, subject to the following conditions:
+--
+-- The above copyright notice and this permission notice shall be included in
+-- all copies or substantial portions of the Software.
+--
+-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+-- FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+-- DEALINGS IN THE SOFTWARE.
+
+-- Records the metrics for the current request.
+--
+-- # Sample StatsD messages
+-- # pub.publisher_name.consumer.consumer_name.app.application_name.service.service_name.prod.region.useast.request.device_links.POST.200.count
+-- # pub.publisher_name.consumer.consumer_name.app.application_name.service.service_name.prod.region.useast.request.device_links.POST.200.responseTime
+-- # pub.publisher_name.consumer.consumer_name.app.application_name.service.service_name.prod.region.useast.request.device_links.POST.200.upstreamResponseTime
+-- # pub.publisher_name.consumer.consumer_name.app.application_name.service.service_name.prod.region.useast.request.validate_request.GET.200.responseTime
+--
+-- NOTE: by default it logs the root-path of the request. If the root path is used for versioning ( i.e. v1.0 ) there is the property $metric_path that
+-- can be set on the location block in order to override the root-path
+--
+-- Created by IntelliJ IDEA.
+-- User: nramaswa
+-- Date: 3/14/14
+-- Time: 12:45 AM
+-- To change this template use File | Settings | File Templates.
+--
+local MetricsCls = require "metrics.MetricsBuffer"
+local metrics = MetricsCls:new()
+
+local M = {}
+
+function M:new(o)
+ o = o or {}
+ setmetatable(o, self)
+ self.__index = self
+ return o
+end
+
+-- To replace . with _
+local function normalizeString(input)
+ return input:gsub("%.", "_")
+end
+
+local function getPathForMetrics(serviceName)
+ local user_defined_path = ngx.var.metrics_path
+ if user_defined_path ~= nil and #user_defined_path < 2 then
+ user_defined_path = nil
+ end
+
+ local request_uri = user_defined_path or ngx.var.uri or "/" .. serviceName
+ local pattern = "(?:\\/?\\/?)([^\\/:]+)" -- To get the first part of the request uri excluding content after :
+ local requestPathFromURI, err = ngx.re.match(request_uri, pattern)
+ local requestPath = serviceName -- default value
+
+ if err then
+ ngx.log(ngx.WARN, "Assigned requestPath as serviceName due to error in extracting requestPathFromURI: ", err)
+ end
+
+ if requestPathFromURI then
+ if requestPathFromURI[1] then
+ requestPath = requestPathFromURI[1]
+ requestPath = requestPath:gsub("%.", "_")
+ --ngx.log(ngx.INFO, "\n the extracted requestPath::"..requestPath)
+ end
+ end
+ return requestPath
+end
+
+function M:logCurrentRequest()
+ -- Variables used in the bucket path
+ local publisherName = ngx.var.publisher_org_name or "undefined"
+ local consumerName = ngx.var.consumer_org_name or "undefined"
+ local appName = ngx.var.app_name or "undefined"
+ local serviceName = ngx.var.service_name or "undefined"
+ local realm = ngx.var.service_env or ngx.var.realm or "sandbox"
+ local requestMethod = ngx.var.request_method or "undefined"
+ local status = ngx.var.status or "0"
+ local validateRequestStatus = ngx.var.validate_request_status or "0"
+
+ -- Values for metrics - converted tonumber() later
+ local bucketCountValue = 1
+ local requestTime = tonumber(ngx.var.request_time) or -1
+ local upstreamResponseTime = tonumber(ngx.var.upstream_response_time) or -1
+ local validateTime = tonumber(ngx.var.validate_request_response_time) or -1
+ local rgnName = ngx.var.aws_region or "undefined"
+ local bytesSent = tonumber(ngx.var.bytes_sent) or -1
+ local bytesReceived = tonumber(ngx.var.request_length) or -1
+
+ -- ---------------------------------------------------------------------- --
+ -- ------------- Logging for all the requests --------------- --
+ -- ---------------------------------------------------------------------- --
+
+ local bucket = "publisher." .. normalizeString(publisherName) ..
+ ".consumer." .. normalizeString(consumerName) ..
+ ".application." .. normalizeString(appName) ..
+ ".service." .. normalizeString(serviceName) ..
+ "." .. realm ..
+ ".region." .. rgnName ..
+ ".request.";
+ local validate_request_response_time = bucket .. "validate_request." .. validateRequestStatus .. ".responseTime";
+
+ local requestPath = getPathForMetrics(serviceName)
+
+ local bytes_sent = bucket .. "bytesSent";
+ local bytes_received = bucket .. "bytesReceived";
+
+
+ --bandwidth data - update only if its greater than zero to sum up all calls
+ if bytesSent > 0 then
+ metrics:count(bytes_sent, bytesSent)
+ end
+ if bytesReceived > 0 then
+ metrics:count(bytes_received, bytesReceived)
+ end
+
+ -- log validate timer entry only if its passed
+ if validateTime >= 0 then
+ metrics:timer(validate_request_response_time, validateTime)
+ end
+
+ -- ---------------------------------------------------------------------- --
+ -- ------------- Non-blocked requests related logging --------------- --
+ -- ---------------------------------------------------------------------- --
+
+ -- Choosing log buckets in statsd based on validation success/failure in the gateway
+ if (validateRequestStatus == "200" or validateRequestStatus == "0" or validateRequestStatus == 200 or validateRequestStatus == "na") then
+ local cc_bucket = bucket ..
+ requestPath .. "." ..
+ requestMethod .. "." .. status;
+
+ local cc_response_time = cc_bucket .. ".responseTime";
+ local cc_upstream_response_time = cc_bucket .. ".upstreamResponseTime";
+
+ local hit_count_for_bucket = cc_bucket .. ".count";
+
+
+ --increament the count for all the calls
+ metrics:count(hit_count_for_bucket, bucketCountValue)
+ -- timers
+ if requestTime >= 0 then
+ metrics:timer(cc_response_time, requestTime)
+ end
+ --timer data - update only if its greater than or = zero as its avging all calls
+ if upstreamResponseTime >= 0 then
+ metrics:timer(cc_upstream_response_time, upstreamResponseTime)
+ end
+ return
+ end
+
+ -- ---------------------------------------------------------------------- --
+ -- ------------- Blocked requests related logging --------------- --
+ -- ---------------------------------------------------------------------- --
+
+ local blocked_bucket = bucket .. "_blocked_"
+
+ local code_count_bucket = blocked_bucket .. "." .. validateRequestStatus .. ".count"
+ local bytes_sent_bucket = blocked_bucket .. ".bytesSent"
+ local bytes_received_bucket = blocked_bucket .. ".bytesReceived"
+
+ --increament the count for all the calls
+ metrics:count(code_count_bucket, bucketCountValue)
+
+ --bandwidth data - update only if its greater than zero to sum up all calls
+ if bytesSent > 0 then
+ metrics:count(bytes_sent_bucket, bytesSent)
+ end
+ if bytesReceived > 0 then
+ metrics:count(bytes_received_bucket, bytesReceived)
+ end
+end
+
+
+return M
+
diff --git a/api-gateway-config/scripts/lua/metrics/factory.lua b/api-gateway-config/scripts/lua/metrics/factory.lua
new file mode 100644
index 0000000..d71d504
--- /dev/null
+++ b/api-gateway-config/scripts/lua/metrics/factory.lua
@@ -0,0 +1,37 @@
+-- Copyright (c) 2015 Adobe Systems Incorporated. All rights reserved.
+--
+-- Permission is hereby granted, free of charge, to any person obtaining a
+-- copy of this software and associated documentation files (the "Software"),
+-- to deal in the Software without restriction, including without limitation
+-- the rights to use, copy, modify, merge, publish, distribute, sublicense,
+-- and/or sell copies of the Software, and to permit persons to whom the
+-- Software is furnished to do so, subject to the following conditions:
+--
+-- The above copyright notice and this permission notice shall be included in
+-- all copies or substantial portions of the Software.
+--
+-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+-- FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+-- DEALINGS IN THE SOFTWARE.
+
+--
+-- A factory to initialize the methods that can be used for capturing usage data
+-- This class is provided as an example
+-- User: ddascal
+--
+
+local MetricsCollector = require "metrics.MetricsCollector"
+local collector = MetricsCollector:new()
+
+local function _captureUsageData()
+ return collector:logCurrentRequest()
+end
+
+return {
+ captureUsageData = _captureUsageData
+}
+
diff --git a/api-gateway-config/scripts/python/logger/StatsdLogger.py b/api-gateway-config/scripts/python/logger/StatsdLogger.py
new file mode 100644
index 0000000..af57f76
--- /dev/null
+++ b/api-gateway-config/scripts/python/logger/StatsdLogger.py
@@ -0,0 +1,93 @@
+#
+# Background worker that loads statistics from the Api-Gateway endpoint and sends them to StatsD.
+# Designed to be restarted every minute by a cronjob
+#
+import json, sys, socket, urllib2
+import argparse
+from threading import Timer
+
+class StatsCollectorWorker:
+
+ UDP_MAX_BUFFER_SIZE = 800
+
+ def __init__(self, statsRestEndpoint, statsd_host, statsd_port ):
+ print(("Starting collecting form [%s], sending to StatsD on [%s:%s]") % (gateway_uri, statsd_host, statsd_port) )
+ self.statsRestEndpoint = statsRestEndpoint
+ self.statsd_host = statsd_host
+ self.statsd_port = statsd_port
+ self.udp_buffer = ""
+
+ def getStatsFromUrl(self, url):
+ try:
+ stats_txt = urllib2.urlopen(url).read()
+ return json.loads(stats_txt)
+ except Exception as e:
+ print "Could not read stats", e
+ return None
+
+ def flushBuffer(self):
+ # print "Sending:", self.udp_buffer
+ self.udp_sock.send(self.udp_buffer)
+ self.udp_buffer = ''
+
+ def sentStatsFromCollection(self, metricCollection, metricType):
+ for i, item in enumerate(metricCollection):
+ # print i, item, metricCollection[item]
+ statsd_metric = ('%s:%s|%s' % (item, metricCollection[item],metricType)).encode("utf-8")
+ if self.udp_buffer.__len__() + statsd_metric.__len__() > StatsCollectorWorker.UDP_MAX_BUFFER_SIZE:
+ self.flushBuffer()
+ self.udp_buffer = ''.join([self.udp_buffer, "\n", statsd_metric])
+
+ def sendStats(self, obj):
+ # TODO: split statsd_host by space in order to support sending data to multiple statsd backends
+ try:
+ self.udp_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+ self.udp_sock.connect((self.statsd_host, self.statsd_port))
+ except Exception as e:
+ print "Could not connect to StatsD", e
+ return None
+ try:
+ self.sentStatsFromCollection(obj["timers"], "ms")
+ self.sentStatsFromCollection(obj["counters"], "c")
+ if self.udp_buffer.__len__() > 0:
+ self.flushBuffer()
+ except Exception as e:
+ print "Could not send statistics", e
+ self.udp_sock.close()
+
+ def getAndSendStats(self,nextInterval, maxRuns):
+ # Read the JSON from the gateway's endpoint
+ # TODO: add an exception in case JSON conversion fails
+ obj = self.getStatsFromUrl(self.statsRestEndpoint)
+ if obj is not None:
+ self.sendStats(obj)
+
+ # print ("Stats processed. %s runs left" % (maxRuns))
+ if maxRuns <= 0:
+ return None
+ t = Timer(nextInterval, self.getAndSendStats, [nextInterval, maxRuns - 1])
+ t.start()
+
+
+statsd_host = "api-gateway-graphite"
+statsd_port = int("8125")
+gateway_uri = "http://127.0.0.1/api-gateway-stats"
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-s', '--statsd-host', dest='statsd_host', required=True)
+ parser.add_argument('-p', '--statsd-port', dest='statsd_port')
+ parser.add_argument('-g', '--gateway-uri', dest='gateway_uri')
+ args = parser.parse_args()
+ statsd_host = args.statsd_host or statsd_host
+ statsd_port = int(args.statsd_port or statsd_port)
+ gateway_uri = args.gateway_uri or gateway_uri
+
+collector = StatsCollectorWorker(statsRestEndpoint=gateway_uri, statsd_host=statsd_host, statsd_port=statsd_port)
+# set the collector to run each 3 seconds , max 18 times
+collector.getAndSendStats(nextInterval=3,maxRuns=18)
+
+# Sample StatsD messages
+# pub.publisher_name.consumer.consumer_name.app.application_name.service.service_name.prod.region.useast.request.device_links.POST.200.count
+# pub.publisher_name.consumer.consumer_name.app.application_name.service.service_name.prod.region.useast.request.device_links.POST.200.responseTime
+# pub.publisher_name.consumer.consumer_name.app.application_name.service.service_name.prod.region.useast.request.device_links.POST.200.upstreamResponseTime
+# pub.publisher_name.consumer.consumer_name.app.application_name.service.service_name.prod.region.useast.request.validate_request.GET.200.responseTime
diff --git a/init.sh b/init.sh
index 0ccf8ca..6876a5d 100755
--- a/init.sh
+++ b/init.sh
@@ -1,4 +1,27 @@
#!/bin/sh
+#/*
+# * Copyright (c) 2012 Adobe Systems Incorporated. All rights reserved.
+# *
+# * Permission is hereby granted, free of charge, to any person obtaining a
+# * copy of this software and associated documentation files (the "Software"),
+# * to deal in the Software without restriction, including without limitation
+# * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# * and/or sell copies of the Software, and to permit persons to whom the
+# * Software is furnished to do so, subject to the following conditions:
+# *
+# * The above copyright notice and this permission notice shall be included in
+# * all copies or substantial portions of the Software.
+# *
+# * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# * DEALINGS IN THE SOFTWARE.
+# *
+# */
+
log_level=${LOG_LEVEL:-warn}
marathon_host=$(echo $MARATHON_HOST)
@@ -12,7 +35,17 @@
if [[ -n "${marathon_host}" ]]; then
echo " ... starting Marathon Service Discovery "
touch /var/run/apigateway-config-watcher.lastrun
+ # start marathon's service discovery
while true; do /etc/api-gateway/marathon-service-discovery.sh > /dev/stderr; sleep 5; done &
+ # start simple statsd logger
+ #
+ # ASSUMPTION: there is a graphite app named "api-gateway-graphite" deployed in marathon
+ #
+ while true; do \
+ statsd_host=$(curl -s ${marathon_host}/v2/apps/api-gateway-graphite/tasks | grep 8125 | awk '{for(i=3;i<=NF;++i) printf("%s ", $i) }' | awk '{for(i=1;i<=NF;++i) sub(/\:\d+/,"",$i); print }' ); \
+ if [[ -n "${statsd_host}" ]]; then python /etc/api-gateway/scripts/python/logger/StatsdLogger.py --statsd-host=${statsd_host} > /var/log/api-gateway/statsd-logger.log; fi; \
+ sleep 60; \
+ done &
fi
echo " ... testing configuration "
