| ### General cluster availability ### |
| |
| # alert if another failed member will result in an unavailable cluster |
| ALERT InsufficientMembers |
| IF count(up{job="etcd"} == 0) > (count(up{job="etcd"}) / 2 - 1) |
| FOR 3m |
| LABELS { |
| severity = "critical" |
| } |
| ANNOTATIONS { |
| summary = "etcd cluster insufficient members", |
| description = "If one more etcd member goes down the cluster will be unavailable", |
| } |
| |
| ### HTTP requests alerts ### |
| |
| # alert if more than 1% of requests to an HTTP endpoint have failed with a non 4xx response |
| ALERT HighNumberOfFailedHTTPRequests |
| IF sum by(method) (rate(etcd_http_failed_total{job="etcd", code!~"4[0-9]{2}"}[5m])) |
| / sum by(method) (rate(etcd_http_received_total{job="etcd"}[5m])) > 0.01 |
| FOR 10m |
| LABELS { |
| severity = "warning" |
| } |
| ANNOTATIONS { |
| summary = "a high number of HTTP requests are failing", |
| description = "{{ $value }}% of requests for {{ $labels.method }} failed on etcd instance {{ $labels.instance }}", |
| } |
| |
| # alert if more than 5% of requests to an HTTP endpoint have failed with a non 4xx response |
| ALERT HighNumberOfFailedHTTPRequests |
| IF sum by(method) (rate(etcd_http_failed_total{job="etcd", code!~"4[0-9]{2}"}[5m])) |
| / sum by(method) (rate(etcd_http_received_total{job="etcd"}[5m])) > 0.05 |
| FOR 5m |
| LABELS { |
| severity = "critical" |
| } |
| ANNOTATIONS { |
| summary = "a high number of HTTP requests are failing", |
| description = "{{ $value }}% of requests for {{ $labels.method }} failed on etcd instance {{ $labels.instance }}", |
| } |
| |
| # alert if 50% of requests get a 4xx response |
| ALERT HighNumberOfFailedHTTPRequests |
| IF sum by(method) (rate(etcd_http_failed_total{job="etcd", code=~"4[0-9]{2}"}[5m])) |
| / sum by(method) (rate(etcd_http_received_total{job="etcd"}[5m])) > 0.5 |
| FOR 10m |
| LABELS { |
| severity = "critical" |
| } |
| ANNOTATIONS { |
| summary = "a high number of HTTP requests are failing", |
| description = "{{ $value }}% of requests for {{ $labels.method }} failed with 4xx responses on etcd instance {{ $labels.instance }}", |
| } |
| |
| # alert if the 99th percentile of HTTP requests take more than 150ms |
| ALERT HTTPRequestsSlow |
| IF histogram_quantile(0.99, rate(etcd_http_successful_duration_second_bucket[5m])) > 0.15 |
| FOR 10m |
| LABELS { |
| severity = "warning" |
| } |
| ANNOTATIONS { |
| summary = "slow HTTP requests", |
| description = "on etcd instance {{ $labels.instance }} HTTP requests to {{ $labels.method }} are slow", |
| } |
| |
| ### File descriptor alerts ### |
| |
| instance:fd_utilization = process_open_fds / process_max_fds |
| |
| # alert if file descriptors are likely to exhaust within the next 4 hours |
| ALERT FdExhaustionClose |
| IF predict_linear(instance:fd_utilization[1h], 3600 * 4) > 1 |
| FOR 10m |
| LABELS { |
| severity = "warning" |
| } |
| ANNOTATIONS { |
| summary = "file descriptors soon exhausted", |
| description = "{{ $labels.job }} instance {{ $labels.instance }} will exhaust its file descriptors soon", |
| } |
| |
| # alert if file descriptors are likely to exhaust within the next hour |
| ALERT FdExhaustionClose |
| IF predict_linear(instance:fd_utilization[10m], 3600) > 1 |
| FOR 10m |
| LABELS { |
| severity = "critical" |
| } |
| ANNOTATIONS { |
| summary = "file descriptors soon exhausted", |
| description = "{{ $labels.job }} instance {{ $labels.instance }} will exhaust its file descriptors soon", |
| } |
| |
| ### etcd proposal alerts ### |
| |
| # alert if there are several failed proposals within an hour |
| ALERT HighNumberOfFailedProposals |
| IF increase(etcd_server_proposal_failed_total{job="etcd"}[1h]) > 5 |
| LABELS { |
| severity = "warning" |
| } |
| ANNOTATIONS { |
| summary = "a high number of proposals within the etcd cluster are failing", |
| description = "etcd instance {{ $labels.instance }} has seen {{ $value }} proposal failures within the last hour", |
| } |
| |
| ### etcd disk io latency alerts ### |
| |
| # alert if 99th percentile of fsync durations is higher than 500ms |
| ALERT HighFsyncDurations |
| IF histogram_quantile(0.99, rate(etcd_wal_fsync_durations_seconds_bucket[5m])) > 0.5 |
| FOR 10m |
| LABELS { |
| severity = "warning" |
| } |
| ANNOTATIONS { |
| summary = "high fsync durations", |
| description = "etcd instance {{ $labels.instance }} fync durations are high", |
| } |