Fix inaccurate Pulsar and Bookkeeper metrics (#12174)
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
index e114bca..a14d03b 100644
--- a/docs/en/changes/changes.md
+++ b/docs/en/changes/changes.md
@@ -119,6 +119,7 @@
* [Break Change] gGRPC metrics exporter unified the metric value type and support labeled metrics.
* Add component definition(ID=152) for `c3p0`(JDBC3 Connection and Statement Pooling).
* Fix MQE `top_n` global query.
+* Fix inaccurate Pulsar and Bookkeeper metrics.
#### UI
diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/bookkeeper/bookkeeper-cluster.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/bookkeeper/bookkeeper-cluster.json
index 68e6c4d..fd7bd5e 100644
--- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/bookkeeper/bookkeeper-cluster.json
+++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/bookkeeper/bookkeeper-cluster.json
@@ -34,7 +34,12 @@
"name": "bookie_ledgers_count"
},
"expressions": [
- "meter_bookkeeper_bookie_ledgers_count"
+ "aggregate_labels(meter_bookkeeper_bookie_ledgers_count,sum)"
+ ],
+ "metricConfig": [
+ {
+ "label": "Ledgers Count"
+ }
]
},
{
@@ -57,7 +62,12 @@
"name": "bookie_ledger_writable_dirs"
},
"expressions": [
- "meter_bookkeeper_bookie_ledger_writable_dirs"
+ "aggregate_labels(meter_bookkeeper_bookie_ledger_writable_dirs,sum)"
+ ],
+ "metricConfig": [
+ {
+ "label": "Ledger Writable Dirs"
+ }
]
},
{
@@ -80,7 +90,12 @@
"name": "bookie_ledger_dir_data_bookkeeper_ledgers_usage"
},
"expressions": [
- "meter_bookkeeper_bookie_ledger_dir_data_bookkeeper_ledgers_usage"
+ "aggregate_labels(meter_bookkeeper_bookie_ledger_dir_data_bookkeeper_ledgers_usage,sum)"
+ ],
+ "metricConfig": [
+ {
+ "label": "Ledger Dir Usage"
+ }
]
},
{
@@ -103,7 +118,12 @@
"name": "bookie_entries_count"
},
"expressions": [
- "meter_bookkeeper_bookie_entries_count"
+ "aggregate_labels(meter_bookkeeper_bookie_entries_count,sum)"
+ ],
+ "metricConfig": [
+ {
+ "label": "Entries Count"
+ }
]
},
{
@@ -126,11 +146,12 @@
"name": "bookie_write_cache_size"
},
"expressions": [
- "meter_bookkeeper_bookie_write_cache_size/1024/1024"
+ "aggregate_labels(meter_bookkeeper_bookie_write_cache_size,sum)/1024/1024"
],
"metricConfig": [
{
- "label": "read cache size"
+ "unit": "MB",
+ "label": "Write Cache Size"
}
]
},
@@ -154,11 +175,11 @@
"name": "bookie_write_cache_count"
},
"expressions": [
- "meter_bookkeeper_bookie_write_cache_count"
+ "aggregate_labels(meter_bookkeeper_bookie_write_cache_count,sum)"
],
"metricConfig": [
{
- "label": "read cache count"
+ "label": "Write Cache Entry Count"
}
]
},
@@ -178,11 +199,17 @@
"showYAxis": true
},
"widget": {
- "title": "Bookie Read Cache (MB)",
+ "title": "Bookie Read Cache Size (MB)",
"name": "_bookie_read_cache_size"
},
"expressions": [
- "meter_bookkeeper_bookie_read_cache_size/1024/1024"
+ "aggregate_labels(meter_bookkeeper_bookie_read_cache_size,sum)/1024/1024"
+ ],
+ "metricConfig": [
+ {
+ "unit": "MB",
+ "label": "Read Cache Size"
+ }
]
},
{
@@ -205,7 +232,12 @@
"name": "throughput_out"
},
"expressions": [
- "meter_bookkeeper_bookie_read_cache_count"
+ "aggregate_labels(meter_bookkeeper_bookie_read_cache_count,sum)"
+ ],
+ "metricConfig": [
+ {
+ "label": "Read Cache Entry Count"
+ }
]
},
{
@@ -216,7 +248,12 @@
"i": "16",
"type": "Widget",
"expressions": [
- "meter_bookkeeper_bookie_read_rate"
+ "aggregate_labels(meter_bookkeeper_bookie_read_rate,sum)"
+ ],
+ "metricConfig": [
+ {
+ "label": "Read Rate"
+ }
],
"graph": {
"type": "Line",
@@ -239,7 +276,12 @@
"i": "18",
"type": "Widget",
"expressions": [
- "meter_bookkeeper_bookie_write_rate"
+ "aggregate_labels(meter_bookkeeper_bookie_write_rate,sum)"
+ ],
+ "metricConfig": [
+ {
+ "label": "Write Rate"
+ }
],
"graph": {
"type": "Line",
diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/bookkeeper/bookkeeper-node.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/bookkeeper/bookkeeper-node.json
index 0ff52cf..c540498 100644
--- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/bookkeeper/bookkeeper-node.json
+++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/bookkeeper/bookkeeper-node.json
@@ -155,7 +155,7 @@
"i": "33",
"type": "Widget",
"expressions": [
- "meter_bookkeeper_node_thread_executor_completed.sum(['cluster','node'])"
+ "meter_bookkeeper_node_thread_executor_completed"
],
"graph": {
"type": "Line",
diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/pulsar/pulsar-broker.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/pulsar/pulsar-broker.json
index f86b157..d08f6b1 100644
--- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/pulsar/pulsar-broker.json
+++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/pulsar/pulsar-broker.json
@@ -82,7 +82,7 @@
"showYAxis": true
},
"expressions": [
- "meter_pulsar_broker_connection_create_fail_count.sum(['cluster','broker'])"
+ "meter_pulsar_broker_connection_create_fail_count"
],
"metricConfig": [
{
diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/pulsar/pulsar-cluster.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/pulsar/pulsar-cluster.json
index fbf00b9..b2e9915 100644
--- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/pulsar/pulsar-cluster.json
+++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/pulsar/pulsar-cluster.json
@@ -35,7 +35,12 @@
"name": "total_topics"
},
"expressions": [
- "meter_pulsar_total_topics"
+ "aggregate_labels(meter_pulsar_total_topics,sum)"
+ ],
+ "metricConfig": [
+ {
+ "label": "Total Topics"
+ }
]
},
{
@@ -59,7 +64,12 @@
"name": "total_subscriptions"
},
"expressions": [
- "meter_pulsar_total_subscriptions"
+ "aggregate_labels(meter_pulsar_total_subscriptions,sum)"
+ ],
+ "metricConfig": [
+ {
+ "label": "Total Subscriptions"
+ }
]
},
{
@@ -83,7 +93,12 @@
"name": "total_producers"
},
"expressions": [
- "meter_pulsar_total_producers"
+ "aggregate_labels(meter_pulsar_total_producers,sum)"
+ ],
+ "metricConfig": [
+ {
+ "label": "Total Producers"
+ }
]
},
{
@@ -107,7 +122,12 @@
"name": "total_consumers"
},
"expressions": [
- "meter_pulsar_total_consumers"
+ "aggregate_labels(meter_pulsar_total_consumers,sum)"
+ ],
+ "metricConfig": [
+ {
+ "label": "Total Consumers"
+ }
]
},
{
@@ -131,7 +151,12 @@
"name": "message_rate_in"
},
"expressions": [
- "meter_pulsar_message_rate_in"
+ "aggregate_labels(meter_pulsar_message_rate_in,sum)"
+ ],
+ "metricConfig": [
+ {
+ "label": "Message Rate In"
+ }
]
},
{
@@ -155,7 +180,12 @@
"name": "message_rate_out"
},
"expressions": [
- "meter_pulsar_message_rate_in"
+ "aggregate_labels(meter_pulsar_message_rate_out,sum)"
+ ],
+ "metricConfig": [
+ {
+ "label": "Message Rate Out"
+ }
]
},
{
@@ -179,7 +209,12 @@
"name": "throughput_in"
},
"expressions": [
- "meter_pulsar_throughput_in"
+ "aggregate_labels(meter_pulsar_throughput_in,sum)"
+ ],
+ "metricConfig": [
+ {
+ "label": "Throughput In"
+ }
]
},
{
@@ -203,7 +238,12 @@
"name": "throughput_out"
},
"expressions": [
- "meter_pulsar_throughput_in"
+ "aggregate_labels(meter_pulsar_throughput_out,sum)"
+ ],
+ "metricConfig": [
+ {
+ "label": "Throughput Out"
+ }
]
},
{
@@ -214,7 +254,12 @@
"i": "15",
"type": "Widget",
"expressions": [
- "meter_pulsar_storage_read_rate.sum(['cluster','node'])"
+ "aggregate_labels(meter_pulsar_storage_read_rate,sum)"
+ ],
+ "metricConfig": [
+ {
+ "label": "Storage Read Rate"
+ }
],
"graph": {
"type": "Line",
@@ -237,7 +282,12 @@
"i": "16",
"type": "Widget",
"expressions": [
- "meter_pulsar_storage_logical_size.sum(['cluster','node'])/1024/1024"
+ "aggregate_labels(meter_pulsar_storage_logical_size,sum)/1024/1024"
+ ],
+ "metricConfig": [
+ {
+ "label": "Storage Logical Size"
+ }
],
"graph": {
"type": "Line",
@@ -260,7 +310,12 @@
"i": "17",
"type": "Widget",
"expressions": [
- "meter_pulsar_storage_write_rate.sum(['cluster','node'])"
+ "aggregate_labels(meter_pulsar_storage_write_rate,sum)"
+ ],
+ "metricConfig": [
+ {
+ "label": "Storage Write Rate"
+ }
],
"graph": {
"type": "Line",
@@ -283,7 +338,12 @@
"i": "18",
"type": "Widget",
"expressions": [
- "meter_pulsar_storage_size/1024/1024"
+ "aggregate_labels(meter_pulsar_storage_size,sum)/1024/1024"
+ ],
+ "metricConfig": [
+ {
+ "label": "Storage Size"
+ }
],
"graph": {
"type": "Line",
@@ -331,11 +391,11 @@
"isRoot": false,
"isDefault": true,
"expressions": [
- "avg(meter_pulsar_total_topics)",
- "avg(meter_pulsar_total_subscriptions)",
- "avg(meter_pulsar_message_rate_in)",
- "avg(meter_pulsar_message_rate_out)",
- "avg(meter_pulsar_storage_size)"
+ "aggregate_labels(meter_pulsar_total_topics,sum)",
+ "aggregate_labels(meter_pulsar_total_subscriptions,sum)",
+ "avg(aggregate_labels(meter_pulsar_message_rate_in,sum))",
+ "avg(aggregate_labels(meter_pulsar_message_rate_out,sum))",
+ "aggregate_labels(meter_pulsar_storage_size,sum)"
],
"expressionsConfig": [
{
@@ -345,13 +405,13 @@
"label": "total subscriptions"
},
{
- "label": "message rate in"
+ "label": "avg message rate in"
},
{
- "label": "message rate out"
+ "label": "avg message rate out"
},
{
- "label": "storage size"
+ "label": "total storage size"
}
]
}
diff --git a/test/e2e-v2/cases/pulsar/otel-collector-config.yaml b/test/e2e-v2/cases/pulsar/otel-collector-config.yaml
index af18c27..4176cc9 100644
--- a/test/e2e-v2/cases/pulsar/otel-collector-config.yaml
+++ b/test/e2e-v2/cases/pulsar/otel-collector-config.yaml
@@ -44,15 +44,18 @@
exporters:
otlp:
- # The OAP Server address
- endpoint: "oap:11800"
+ endpoint: oap:11800
tls:
insecure: true
- debug:
- verbosity: detailed
+ logging:
+ loglevel: debug
service:
pipelines:
metrics:
- receivers: [prometheus]
- processors: [batch]
- exporters: [debug,otlp]
\ No newline at end of file
+ receivers:
+ - prometheus
+ processors:
+ - batch
+ exporters:
+ - otlp
+ - logging
\ No newline at end of file
diff --git a/test/e2e-v2/cases/pulsar/pulsar-cases.yaml b/test/e2e-v2/cases/pulsar/pulsar-cases.yaml
index 091932e..e450720 100644
--- a/test/e2e-v2/cases/pulsar/pulsar-cases.yaml
+++ b/test/e2e-v2/cases/pulsar/pulsar-cases.yaml
@@ -22,30 +22,30 @@
- query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance ls --service-name=bookkeeper::pulsar-cluster
expected: expected/bookie_instance.yml
# service cases for pulsar-cluster
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_pulsar_total_topics --service-name=pulsar::pulsar-cluster
- expected: expected/metrics-has-value-instance-broker-label.yml
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_pulsar_total_subscriptions --service-name=pulsar::pulsar-cluster
- expected: expected/metrics-has-value-instance-broker-label.yml
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_pulsar_total_producers --service-name=pulsar::pulsar-cluster
- expected: expected/metrics-has-value-instance-broker-label.yml
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_pulsar_total_consumers --service-name=pulsar::pulsar-cluster
- expected: expected/metrics-has-value-instance-broker-label.yml
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_pulsar_message_rate_in --service-name=pulsar::pulsar-cluster
- expected: expected/metrics-has-value-instance-broker-label.yml
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_pulsar_message_rate_out --service-name=pulsar::pulsar-cluster
- expected: expected/metrics-has-value-instance-broker-label.yml
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_pulsar_throughput_in --service-name=pulsar::pulsar-cluster
- expected: expected/metrics-has-value-instance-broker-label.yml
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_pulsar_throughput_out --service-name=pulsar::pulsar-cluster
- expected: expected/metrics-has-value-instance-broker-label.yml
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_pulsar_storage_size --service-name=pulsar::pulsar-cluster
- expected: expected/metrics-has-value-instance-broker-label.yml
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_pulsar_storage_logical_size --service-name=pulsar::pulsar-cluster
- expected: expected/metrics-has-value-instance-broker-label.yml
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_pulsar_storage_write_rate --service-name=pulsar::pulsar-cluster
- expected: expected/metrics-has-value-instance-broker-label.yml
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_pulsar_storage_read_rate --service-name=pulsar::pulsar-cluster
- expected: expected/metrics-has-value-instance-broker-label.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression='aggregate_labels(meter_pulsar_total_topics,sum)' --service-name=pulsar::pulsar-cluster
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression='aggregate_labels(meter_pulsar_total_subscriptions,sum)' --service-name=pulsar::pulsar-cluster
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression='aggregate_labels(meter_pulsar_total_producers,sum)' --service-name=pulsar::pulsar-cluster
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression='aggregate_labels(meter_pulsar_total_consumers,sum)' --service-name=pulsar::pulsar-cluster
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression='aggregate_labels(meter_pulsar_message_rate_in,sum)' --service-name=pulsar::pulsar-cluster
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression='aggregate_labels(meter_pulsar_message_rate_out,sum)' --service-name=pulsar::pulsar-cluster
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression='aggregate_labels(meter_pulsar_throughput_in,sum)' --service-name=pulsar::pulsar-cluster
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression='aggregate_labels(meter_pulsar_throughput_out,sum)' --service-name=pulsar::pulsar-cluster
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression='aggregate_labels(meter_pulsar_storage_size,sum)' --service-name=pulsar::pulsar-cluster
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression='aggregate_labels(meter_pulsar_storage_logical_size,sum)' --service-name=pulsar::pulsar-cluster
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression='aggregate_labels(meter_pulsar_storage_write_rate,sum)' --service-name=pulsar::pulsar-cluster
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression='aggregate_labels(meter_pulsar_storage_read_rate,sum)' --service-name=pulsar::pulsar-cluster
+ expected: expected/metrics-has-value.yml
# service cases for pulsar-broker
- query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_pulsar_broker_active_connections --service-name=pulsar::pulsar-cluster
@@ -74,26 +74,26 @@
expected: expected/metrics-has-value.yml
# service cases for the bookkeeper cluster
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_bookkeeper_bookie_ledgers_count --service-name=bookkeeper::pulsar-cluster
- expected: expected/metrics-has-value-instance-bookie-label.yml
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_bookkeeper_bookie_ledger_writable_dirs --service-name=bookkeeper::pulsar-cluster
- expected: expected/metrics-has-value-instance-bookie-label.yml
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_bookkeeper_bookie_ledger_dir_data_bookkeeper_ledgers_usage --service-name=bookkeeper::pulsar-cluster
- expected: expected/metrics-has-value-instance-bookie-label.yml
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_bookkeeper_bookie_entries_count --service-name=bookkeeper::pulsar-cluster
- expected: expected/metrics-has-value-instance-bookie-label.yml
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_bookkeeper_bookie_write_cache_size --service-name=bookkeeper::pulsar-cluster
- expected: expected/metrics-has-value-instance-bookie-label.yml
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_bookkeeper_bookie_write_cache_count --service-name=bookkeeper::pulsar-cluster
- expected: expected/metrics-has-value-instance-bookie-label.yml
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_bookkeeper_bookie_read_cache_size --service-name=bookkeeper::pulsar-cluster
- expected: expected/metrics-has-value-instance-bookie-label.yml
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_bookkeeper_bookie_read_cache_count --service-name=bookkeeper::pulsar-cluster
- expected: expected/metrics-has-value-instance-bookie-label.yml
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_bookkeeper_bookie_write_rate --service-name=bookkeeper::pulsar-cluster
- expected: expected/metrics-has-value-instance-bookie-label.yml
- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_bookkeeper_bookie_read_rate --service-name=bookkeeper::pulsar-cluster
- expected: expected/metrics-has-value-instance-bookie-label.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression='aggregate_labels(meter_bookkeeper_bookie_ledgers_count,sum)' --service-name=bookkeeper::pulsar-cluster
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression='aggregate_labels(meter_bookkeeper_bookie_ledger_writable_dirs,sum)' --service-name=bookkeeper::pulsar-cluster
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression='aggregate_labels(meter_bookkeeper_bookie_ledger_dir_data_bookkeeper_ledgers_usage,sum)' --service-name=bookkeeper::pulsar-cluster
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression='aggregate_labels(meter_bookkeeper_bookie_entries_count,sum)' --service-name=bookkeeper::pulsar-cluster
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression='aggregate_labels(meter_bookkeeper_bookie_write_cache_size,sum)' --service-name=bookkeeper::pulsar-cluster
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression='aggregate_labels(meter_bookkeeper_bookie_write_cache_count,sum)' --service-name=bookkeeper::pulsar-cluster
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression='aggregate_labels(meter_bookkeeper_bookie_read_cache_size,sum)' --service-name=bookkeeper::pulsar-cluster
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression='aggregate_labels(meter_bookkeeper_bookie_read_cache_count,sum)' --service-name=bookkeeper::pulsar-cluster
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression='aggregate_labels(meter_bookkeeper_bookie_write_rate,sum)' --service-name=bookkeeper::pulsar-cluster
+ expected: expected/metrics-has-value.yml
+ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression='aggregate_labels(meter_bookkeeper_bookie_read_rate,sum)' --service-name=bookkeeper::pulsar-cluster
+ expected: expected/metrics-has-value.yml
# service cases for the bookkeeper node
