Add Prometheus metrics rules (#3651)
diff --git a/heron/config/src/yaml/conf/aurora/metrics_sinks.yaml b/heron/config/src/yaml/conf/aurora/metrics_sinks.yaml
index 6530209..6cac2f1 100644
--- a/heron/config/src/yaml/conf/aurora/metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/aurora/metrics_sinks.yaml
@@ -114,6 +114,137 @@
# include-topology-name: true # Include topology name in metric name (default false)
# metrics-cache-max-size: 1000000 # Max number of metrics cached and published (default 1000000)
# metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was collected will stopped being published (default 600)
+# rules:
+# # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
+# - pattern: __jvm-(.+)/(.+)
+# name: jvm_$1_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# # "__execute-time-ns/pulsar-prod-4/default": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$2"
+# context: "$3"
+# # "__execute-time-ns/pulsar-prod-4": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# context: "$2"
+# # StreamManager
+# # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
+# - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
+# name: $2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$1"
+# # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes": "0.000000",
+# - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
+# name: $1_$4
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
+# - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # PulsarSpoutMetrics of PulsarSpout 'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
+# - pattern: PulsarSpout/(.+)/(.+)
+# name: pulsar_spout_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# - pattern: PulsarBolt/(.+)/(.+)
+# name: pulsar_bolt_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# # name: "kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# node_id: "$4"
+# # name: "kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
+# - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# # name: "kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# partition: "$5"
+# # name: "kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
+# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# # name: "kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# node_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
+# - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
+# name: kafka_offset_$3
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
+# partition: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
+# - pattern: kafkaOffset/(.+)/(.+)
+# name: kafka_offset_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
### Config for graphite-sink
### Currently the graphite-sink is disabled
diff --git a/heron/config/src/yaml/conf/examples/metrics_sinks.yaml b/heron/config/src/yaml/conf/examples/metrics_sinks.yaml
index 9d36164..cfcc7f0 100644
--- a/heron/config/src/yaml/conf/examples/metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/examples/metrics_sinks.yaml
@@ -114,6 +114,137 @@
# include-topology-name: true # Include topology name in metric name (default false)
# metrics-cache-max-size: 1000000 # Max number of metrics cached and published (default 1000000)
# metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was collected will stopped being published (default 600)
+# rules:
+# # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
+# - pattern: __jvm-(.+)/(.+)
+# name: jvm_$1_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# # "__execute-time-ns/pulsar-prod-4/default": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$2"
+# context: "$3"
+# # "__execute-time-ns/pulsar-prod-4": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# context: "$2"
+# # StreamManager
+# # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
+# - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
+# name: $2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$1"
+# # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes": "0.000000",
+# - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
+# name: $1_$4
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
+# - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # PulsarSpoutMetrics of PulsarSpout 'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
+# - pattern: PulsarSpout/(.+)/(.+)
+# name: pulsar_spout_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# - pattern: PulsarBolt/(.+)/(.+)
+# name: pulsar_bolt_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# # name: "kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# node_id: "$4"
+# # name: "kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
+# - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# # name: "kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# partition: "$5"
+# # name: "kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
+# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# # name: "kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# node_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
+# - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
+# name: kafka_offset_$3
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
+# partition: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
+# - pattern: kafkaOffset/(.+)/(.+)
+# name: kafka_offset_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
### Config for graphite-sink
### Currently the graphite-sink is disabled
diff --git a/heron/config/src/yaml/conf/kubernetes/metrics_sinks.yaml b/heron/config/src/yaml/conf/kubernetes/metrics_sinks.yaml
index d6c5a39..081f5c7 100644
--- a/heron/config/src/yaml/conf/kubernetes/metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/kubernetes/metrics_sinks.yaml
@@ -76,13 +76,144 @@
"__time_spent_back_pressure_by_compid": SUM
prometheus-sink:
- class: "org.apache.heron.metricsmgr.sink.PrometheusSink"
- port: 8080 # The port on which to run (either port or port-file are mandatory)
- path: /metrics # The path on which to publish the metrics (mandatory)
- flat-metrics: true # By default the web-sink will publish a flat "name -> value" json map
- include-topology-name: true # Include topology name in metric name (default false)
- metrics-cache-max-size: 1000000 # Max number of metrics cached and published (default 1000000)
- metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was collected will stopped being published (default 600)
+ class: "org.apache.heron.metricsmgr.sink.PrometheusSink"
+ port: 8080 # The port on which to run (either port or port-file are mandatory)
+ path: /metrics # The path on which to publish the metrics (mandatory)
+ flat-metrics: true # By default the web-sink will publish a flat "name -> value" json map
+ include-topology-name: true # Include topology name in metric name (default false)
+ metrics-cache-max-size: 1000000 # Max number of metrics cached and published (default 1000000)
+ metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was collected will stopped being published (default 600)
+ rules:
+ # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
+ - pattern: __jvm-(.+)/(.+)
+ name: jvm_$1_$2
+ attrNameSnakeCase: true
+ type: COUNTER
+ # "__execute-time-ns/pulsar-prod-4/default": "418764",
+ - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
+ name: $1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ dest_component: "$2"
+ context: "$3"
+ # "__execute-time-ns/pulsar-prod-4": "418764",
+ - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
+ name: $1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ context: "$2"
+ # StreamManager
+ # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
+ - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
+ name: $2
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ dest_component: "$1"
+ # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes": "0.000000",
+ - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
+ name: $1_$4
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ dest_container: "$2"
+ dest_task: "$3"
+ # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
+ - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
+ name: $1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ dest_container: "$2"
+ dest_task: "$3"
+ # PulsarSpoutMetrics of PulsarSpout 'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
+ - pattern: PulsarSpout/(.+)/(.+)
+ name: pulsar_spout_$2
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ component: "$1"
+ - pattern: PulsarBolt/(.+)/(.+)
+ name: pulsar_bolt_$2
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ component: "$1"
+ # name: "kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
+ - pattern: kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
+ name: kafka_consumer_$2_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$3"
+ node_id: "$4"
+ # name: "kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
+ - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
+ name: kafka_consumer_$2_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$3"
+ # name: "kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
+ - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
+ name: kafka_consumer_$2_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$3"
+ topic: "$4"
+ partition: "$5"
+ # name: "kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
+ - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
+ name: kafka_consumer_$2_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$3"
+ topic: "$4"
+ # name: "kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
+ - pattern: kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
+ name: kafka_consumer_$2_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$3"
+ - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
+ name: kafka_consumer_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$2"
+ node_id: "$3"
+ - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
+ name: kafka_consumer_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$2"
+ - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
+ name: kafka_consumer_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$2"
+ # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
+ - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
+ name: kafka_offset_$3
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ topic: "$1"
+ partition: "$2"
+ # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
+ - pattern: kafkaOffset/(.+)/(.+)
+ name: kafka_offset_$2
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ topic: "$1"
### Config for metricscache-sink
metricscache-sink:
diff --git a/heron/config/src/yaml/conf/local/metrics_sinks.yaml b/heron/config/src/yaml/conf/local/metrics_sinks.yaml
index 6530209..6cac2f1 100644
--- a/heron/config/src/yaml/conf/local/metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/local/metrics_sinks.yaml
@@ -114,6 +114,137 @@
# include-topology-name: true # Include topology name in metric name (default false)
# metrics-cache-max-size: 1000000 # Max number of metrics cached and published (default 1000000)
# metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was collected will stopped being published (default 600)
+# rules:
+# # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
+# - pattern: __jvm-(.+)/(.+)
+# name: jvm_$1_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# # "__execute-time-ns/pulsar-prod-4/default": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$2"
+# context: "$3"
+# # "__execute-time-ns/pulsar-prod-4": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# context: "$2"
+# # StreamManager
+# # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
+# - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
+# name: $2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$1"
+# # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes": "0.000000",
+# - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
+# name: $1_$4
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
+# - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # PulsarSpoutMetrics of PulsarSpout 'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
+# - pattern: PulsarSpout/(.+)/(.+)
+# name: pulsar_spout_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# - pattern: PulsarBolt/(.+)/(.+)
+# name: pulsar_bolt_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# # name: "kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# node_id: "$4"
+# # name: "kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
+# - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# # name: "kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# partition: "$5"
+# # name: "kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
+# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# # name: "kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# node_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
+# - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
+# name: kafka_offset_$3
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
+# partition: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
+# - pattern: kafkaOffset/(.+)/(.+)
+# name: kafka_offset_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
### Config for graphite-sink
### Currently the graphite-sink is disabled
diff --git a/heron/config/src/yaml/conf/localzk/metrics_sinks.yaml b/heron/config/src/yaml/conf/localzk/metrics_sinks.yaml
index 6530209..6cac2f1 100644
--- a/heron/config/src/yaml/conf/localzk/metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/localzk/metrics_sinks.yaml
@@ -114,6 +114,137 @@
# include-topology-name: true # Include topology name in metric name (default false)
# metrics-cache-max-size: 1000000 # Max number of metrics cached and published (default 1000000)
# metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was collected will stopped being published (default 600)
+# rules:
+# # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
+# - pattern: __jvm-(.+)/(.+)
+# name: jvm_$1_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# # "__execute-time-ns/pulsar-prod-4/default": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$2"
+# context: "$3"
+# # "__execute-time-ns/pulsar-prod-4": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# context: "$2"
+# # StreamManager
+# # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
+# - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
+# name: $2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$1"
+# # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes": "0.000000",
+# - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
+# name: $1_$4
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
+# - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # PulsarSpoutMetrics of PulsarSpout 'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
+# - pattern: PulsarSpout/(.+)/(.+)
+# name: pulsar_spout_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# - pattern: PulsarBolt/(.+)/(.+)
+# name: pulsar_bolt_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# # name: "kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# node_id: "$4"
+# # name: "kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
+# - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# # name: "kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# partition: "$5"
+# # name: "kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
+# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# # name: "kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# node_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
+# - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
+# name: kafka_offset_$3
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
+# partition: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
+# - pattern: kafkaOffset/(.+)/(.+)
+# name: kafka_offset_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
### Config for graphite-sink
### Currently the graphite-sink is disabled
diff --git a/heron/config/src/yaml/conf/marathon/metrics_sinks.yaml b/heron/config/src/yaml/conf/marathon/metrics_sinks.yaml
index 6530209..6cac2f1 100644
--- a/heron/config/src/yaml/conf/marathon/metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/marathon/metrics_sinks.yaml
@@ -114,6 +114,137 @@
# include-topology-name: true # Include topology name in metric name (default false)
# metrics-cache-max-size: 1000000 # Max number of metrics cached and published (default 1000000)
# metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was collected will stopped being published (default 600)
+# rules:
+# # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
+# - pattern: __jvm-(.+)/(.+)
+# name: jvm_$1_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# # "__execute-time-ns/pulsar-prod-4/default": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$2"
+# context: "$3"
+# # "__execute-time-ns/pulsar-prod-4": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# context: "$2"
+# # StreamManager
+# # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
+# - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
+# name: $2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$1"
+# # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes": "0.000000",
+# - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
+# name: $1_$4
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
+# - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # PulsarSpoutMetrics of PulsarSpout 'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
+# - pattern: PulsarSpout/(.+)/(.+)
+# name: pulsar_spout_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# - pattern: PulsarBolt/(.+)/(.+)
+# name: pulsar_bolt_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# # name: "kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# node_id: "$4"
+# # name: "kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
+# - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# # name: "kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# partition: "$5"
+# # name: "kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
+# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# # name: "kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# node_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
+# - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
+# name: kafka_offset_$3
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
+# partition: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
+# - pattern: kafkaOffset/(.+)/(.+)
+# name: kafka_offset_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
### Config for graphite-sink
### Currently the graphite-sink is disabled
diff --git a/heron/config/src/yaml/conf/mesos/metrics_sinks.yaml b/heron/config/src/yaml/conf/mesos/metrics_sinks.yaml
index 6530209..6cac2f1 100644
--- a/heron/config/src/yaml/conf/mesos/metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/mesos/metrics_sinks.yaml
@@ -114,6 +114,137 @@
# include-topology-name: true # Include topology name in metric name (default false)
# metrics-cache-max-size: 1000000 # Max number of metrics cached and published (default 1000000)
# metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was collected will stopped being published (default 600)
+# rules:
+# # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
+# - pattern: __jvm-(.+)/(.+)
+# name: jvm_$1_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# # "__execute-time-ns/pulsar-prod-4/default": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$2"
+# context: "$3"
+# # "__execute-time-ns/pulsar-prod-4": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# context: "$2"
+# # StreamManager
+# # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
+# - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
+# name: $2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$1"
+# # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes": "0.000000",
+# - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
+# name: $1_$4
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
+# - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # PulsarSpoutMetrics of PulsarSpout 'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
+# - pattern: PulsarSpout/(.+)/(.+)
+# name: pulsar_spout_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# - pattern: PulsarBolt/(.+)/(.+)
+# name: pulsar_bolt_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# # name: "kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# node_id: "$4"
+# # name: "kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
+# - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# # name: "kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# partition: "$5"
+# # name: "kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
+# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# # name: "kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# node_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
+# - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
+# name: kafka_offset_$3
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
+# partition: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
+# - pattern: kafkaOffset/(.+)/(.+)
+# name: kafka_offset_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
### Config for graphite-sink
### Currently the graphite-sink is disabled
diff --git a/heron/config/src/yaml/conf/nomad/metrics_sinks.yaml b/heron/config/src/yaml/conf/nomad/metrics_sinks.yaml
index 6a3c4fb..2e7d426 100644
--- a/heron/config/src/yaml/conf/nomad/metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/nomad/metrics_sinks.yaml
@@ -84,6 +84,137 @@
include-topology-name: true # Include topology name in metric name (default false)
metrics-cache-max-size: 1000000 # Max number of metrics cached and published (default 1000000)
metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was collected will stopped being published (default 600)
+ rules:
+ # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
+ - pattern: __jvm-(.+)/(.+)
+ name: jvm_$1_$2
+ attrNameSnakeCase: true
+ type: COUNTER
+ # "__execute-time-ns/pulsar-prod-4/default": "418764",
+ - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
+ name: $1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ dest_component: "$2"
+ context: "$3"
+ # "__execute-time-ns/pulsar-prod-4": "418764",
+ - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
+ name: $1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ context: "$2"
+ # StreamManager
+ # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
+ - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
+ name: $2
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ dest_component: "$1"
+ # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes": "0.000000",
+ - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
+ name: $1_$4
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ dest_container: "$2"
+ dest_task: "$3"
+ # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
+ - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
+ name: $1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ dest_container: "$2"
+ dest_task: "$3"
+ # PulsarSpoutMetrics of PulsarSpout 'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
+ - pattern: PulsarSpout/(.+)/(.+)
+ name: pulsar_spout_$2
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ component: "$1"
+ - pattern: PulsarBolt/(.+)/(.+)
+ name: pulsar_bolt_$2
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ component: "$1"
+ # name: "kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
+ - pattern: kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
+ name: kafka_consumer_$2_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$3"
+ node_id: "$4"
+ # name: "kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
+ - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
+ name: kafka_consumer_$2_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$3"
+ # name: "kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
+ - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
+ name: kafka_consumer_$2_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$3"
+ topic: "$4"
+ partition: "$5"
+ # name: "kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
+ - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
+ name: kafka_consumer_$2_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$3"
+ topic: "$4"
+ # name: "kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
+ - pattern: kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
+ name: kafka_consumer_$2_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$3"
+ - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
+ name: kafka_consumer_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$2"
+ node_id: "$3"
+ - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
+ name: kafka_consumer_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$2"
+ - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
+ name: kafka_consumer_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$2"
+ # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
+ - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
+ name: kafka_offset_$3
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ topic: "$1"
+ partition: "$2"
+ # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
+ - pattern: kafkaOffset/(.+)/(.+)
+ name: kafka_offset_$2
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ topic: "$1"
### Config for metricscache-sink
metricscache-sink:
diff --git a/heron/config/src/yaml/conf/sandbox/metrics_sinks.yaml b/heron/config/src/yaml/conf/sandbox/metrics_sinks.yaml
index 6530209..6cac2f1 100644
--- a/heron/config/src/yaml/conf/sandbox/metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/sandbox/metrics_sinks.yaml
@@ -114,6 +114,137 @@
# include-topology-name: true # Include topology name in metric name (default false)
# metrics-cache-max-size: 1000000 # Max number of metrics cached and published (default 1000000)
# metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was collected will stopped being published (default 600)
+# rules:
+# # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
+# - pattern: __jvm-(.+)/(.+)
+# name: jvm_$1_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# # "__execute-time-ns/pulsar-prod-4/default": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$2"
+# context: "$3"
+# # "__execute-time-ns/pulsar-prod-4": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# context: "$2"
+# # StreamManager
+# # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
+# - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
+# name: $2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$1"
+# # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes": "0.000000",
+# - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
+# name: $1_$4
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
+# - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # PulsarSpoutMetrics of PulsarSpout 'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
+# - pattern: PulsarSpout/(.+)/(.+)
+# name: pulsar_spout_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# - pattern: PulsarBolt/(.+)/(.+)
+# name: pulsar_bolt_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# # name: "kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# node_id: "$4"
+# # name: "kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
+# - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# # name: "kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# partition: "$5"
+# # name: "kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
+# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# # name: "kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# node_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
+# - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
+# name: kafka_offset_$3
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
+# partition: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
+# - pattern: kafkaOffset/(.+)/(.+)
+# name: kafka_offset_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
### Config for graphite-sink
### Currently the graphite-sink is disabled
diff --git a/heron/config/src/yaml/conf/slurm/metrics_sinks.yaml b/heron/config/src/yaml/conf/slurm/metrics_sinks.yaml
index 6530209..6cac2f1 100644
--- a/heron/config/src/yaml/conf/slurm/metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/slurm/metrics_sinks.yaml
@@ -114,6 +114,137 @@
# include-topology-name: true # Include topology name in metric name (default false)
# metrics-cache-max-size: 1000000 # Max number of metrics cached and published (default 1000000)
# metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was collected will stopped being published (default 600)
+# rules:
+# # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
+# - pattern: __jvm-(.+)/(.+)
+# name: jvm_$1_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# # "__execute-time-ns/pulsar-prod-4/default": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$2"
+# context: "$3"
+# # "__execute-time-ns/pulsar-prod-4": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# context: "$2"
+# # StreamManager
+# # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
+# - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
+# name: $2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$1"
+# # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes": "0.000000",
+# - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
+# name: $1_$4
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
+# - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # PulsarSpoutMetrics of PulsarSpout 'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
+# - pattern: PulsarSpout/(.+)/(.+)
+# name: pulsar_spout_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# - pattern: PulsarBolt/(.+)/(.+)
+# name: pulsar_bolt_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# # name: "kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# node_id: "$4"
+# # name: "kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
+# - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# # name: "kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# partition: "$5"
+# # name: "kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
+# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# # name: "kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# node_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
+# - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
+# name: kafka_offset_$3
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
+# partition: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
+# - pattern: kafkaOffset/(.+)/(.+)
+# name: kafka_offset_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
### Config for graphite-sink
### Currently the graphite-sink is disabled
diff --git a/heron/config/src/yaml/conf/test/test_metrics_sinks.yaml b/heron/config/src/yaml/conf/test/test_metrics_sinks.yaml
index 6ae89b3..21add70 100644
--- a/heron/config/src/yaml/conf/test/test_metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/test/test_metrics_sinks.yaml
@@ -111,6 +111,137 @@
# include-topology-name: true # Include topology name in metric name (default false)
# metrics-cache-max-size: 1000000 # Max number of metrics cached and published (default 1000000)
# metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was collected will stopped being published (default 600)
+# rules:
+# # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
+# - pattern: __jvm-(.+)/(.+)
+# name: jvm_$1_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# # "__execute-time-ns/pulsar-prod-4/default": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$2"
+# context: "$3"
+# # "__execute-time-ns/pulsar-prod-4": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# context: "$2"
+# # StreamManager
+# # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
+# - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
+# name: $2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$1"
+# # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes": "0.000000",
+# - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
+# name: $1_$4
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
+# - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # PulsarSpoutMetrics of PulsarSpout 'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
+# - pattern: PulsarSpout/(.+)/(.+)
+# name: pulsar_spout_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# - pattern: PulsarBolt/(.+)/(.+)
+# name: pulsar_bolt_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# # name: "kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# node_id: "$4"
+# # name: "kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
+# - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# # name: "kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# partition: "$5"
+# # name: "kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
+# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# # name: "kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# node_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
+# - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
+# name: kafka_offset_$3
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
+# partition: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
+# - pattern: kafkaOffset/(.+)/(.+)
+# name: kafka_offset_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
### Config for scribe-sink
# scribe-sink:
diff --git a/heron/config/src/yaml/conf/yarn/metrics_sinks.yaml b/heron/config/src/yaml/conf/yarn/metrics_sinks.yaml
index 6530209..6cac2f1 100644
--- a/heron/config/src/yaml/conf/yarn/metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/yarn/metrics_sinks.yaml
@@ -114,6 +114,137 @@
# include-topology-name: true # Include topology name in metric name (default false)
# metrics-cache-max-size: 1000000 # Max number of metrics cached and published (default 1000000)
# metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was collected will stopped being published (default 600)
+# rules:
+# # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
+# - pattern: __jvm-(.+)/(.+)
+# name: jvm_$1_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# # "__execute-time-ns/pulsar-prod-4/default": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$2"
+# context: "$3"
+# # "__execute-time-ns/pulsar-prod-4": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# context: "$2"
+# # StreamManager
+# # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
+# - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
+# name: $2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$1"
+# # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes": "0.000000",
+# - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
+# name: $1_$4
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
+# - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # PulsarSpoutMetrics of PulsarSpout 'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
+# - pattern: PulsarSpout/(.+)/(.+)
+# name: pulsar_spout_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# - pattern: PulsarBolt/(.+)/(.+)
+# name: pulsar_bolt_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# # name: "kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# node_id: "$4"
+# # name: "kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
+# - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# # name: "kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# partition: "$5"
+# # name: "kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
+# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# # name: "kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
+# - pattern: kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# node_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
+# - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
+# name: kafka_offset_$3
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
+# partition: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
+# - pattern: kafkaOffset/(.+)/(.+)
+# name: kafka_offset_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
### Config for graphite-sink
### Currently the graphite-sink is disabled
