Alerts

alert: PolarDB-X CDC delay is High
expr: sum by(pod, polardbx_name, namespace) (polardbx_cdc_dumper_delay_in_millisecond) >= 600000
for: 5m
labels:
  severity: warning
annotations:
  description: PolarDB-X {{ $labels.polardbx_name}} pod {{ $labels.pod }} namespace {{ $labels.namespace }} cdc delay is high, over 10min.
  summary: PolarDB-X CDC delay is High

alert: PolarDB-X CDC delay is Too High
expr: sum by(pod, polardbx_name, namespace) (polardbx_cdc_dumper_delay_in_millisecond) >= 1.8e+06
for: 5m
labels:
  severity: critical
annotations:
  description: PolarDB-X {{ $labels.polardbx_name}} pod {{ $labels.pod }} namespace {{ $labels.namespace }} cdc delay is too high, over 30min.
  summary: PolarDB-X CDC delay is Too High

alert: PolarDB-X CN Error SQL Count is High
expr: sum by(pod, polardbx_name, namespace) (rate(polardbx_stats_error_count_total[1m])) >= 20
for: 5m
labels:
  severity: warning
annotations:
  description: PolarDB-X {{ $labels.polardbx_name}} pod {{ $labels.pod }} namespace {{ $labels.namespace }} error sql count is high, over 20 for 5m.
  summary: PolarDB-X CN Error SQL Count is High

alert: PolarDB-X CN Error SQL Count is Too High
expr: sum by(pod, polardbx_name, namespace) (rate(polardbx_stats_error_count_total[1m])) >= 50
for: 5m
labels:
  severity: critical
annotations:
  description: PolarDB-X {{ $labels.polardbx_name}} pod {{ $labels.pod }} namespace {{ $labels.namespace }} error sql count is too high, over 50 for 5m.
  summary: PolarDB-X CN Error SQL Count is Too High

alert: PolarDB-X CN SQL Running Connection is High
expr: sum by(pod, polardbx_name, namespace) (polardbx_stats_running_count) >= 500
for: 5m
labels:
  severity: warning
annotations:
  description: PolarDB-X {{ $labels.polardbx_name}} pod {{ $labels.pod }} namespace {{ $labels.namespace }} running count is high, over 500 for 5m.
  summary: PolarDB-X CN SQL Running Connection is High

alert: PolarDB-X CN SQL Running Connection is Too High
expr: sum by(pod, polardbx_name, namespace) (polardbx_stats_running_count) >= 1000
for: 5m
labels:
  severity: critical
annotations:
  description: PolarDB-X {{ $labels.polardbx_name}} pod {{ $labels.pod }} namespace {{ $labels.namespace }} running count is too high, over 1000 for 5m.
  summary: PolarDB-X CN SQL Running Connection is Too High

alert: PolarDB-X CN Slow SQL Count is High
expr: sum by(pod, polardbx_name, namespace) (rate(polardbx_stats_slow_request_count_total[1m])) >= 200
for: 5m
labels:
  severity: warning
annotations:
  description: PolarDB-X {{ $labels.polardbx_name}} pod {{ $labels.pod }} namespace {{ $labels.namespace }} slow sql count is high, over 200 for 5m.
  summary: PolarDB-X CN Slow SQL Count is High

alert: PolarDB-X CN Slow SQL Count is Too High
expr: sum by(pod, polardbx_name, namespace) (rate(polardbx_stats_slow_request_count_total[1m])) >= 250
for: 5m
labels:
  severity: critical
annotations:
  description: PolarDB-X {{ $labels.polardbx_name}} pod {{ $labels.pod }} namespace {{ $labels.namespace }} slow sql count is too high, over 250 for 5m.
  summary: PolarDB-X CN Slow SQL Count is too High

alert: PolarDB-X CN Total Connection is High
expr: sum by(pod, polardbx_name, namespace) (polardbx_stats_active_connections) >= 5000
for: 5m
labels:
  severity: warning
annotations:
  description: PolarDB-X {{ $labels.polardbx_name}} pod {{ $labels.pod }} namespace {{ $labels.namespace }} total connection is high, over 5000.
  summary: PolarDB-X CN Total Connection is High

alert: PolarDB-X CN Total Connection is Too High
expr: sum by(pod, polardbx_name, namespace) (polardbx_stats_active_connections) >= 10000
for: 5m
labels:
  severity: critical
annotations:
  description: PolarDB-X {{ $labels.polardbx_name}} pod {{ $labels.pod }} namespace {{ $labels.namespace }} total connection is too high, over 10000.
  summary: PolarDB-X CN Total Connection is Too High

alert: PolarDB-X DN Connection Usage is High
expr: sum by(pod, polardbx_name, namespace) (100 * mysql_global_status_threads_connected / mysql_global_variables_max_connections{polardbx_role="dn"}) >= 80
for: 5m
labels:
  severity: warning
annotations:
  description: PolarDB-X {{ $labels.polardbx_name}} pod {{ $labels.pod }} namespace {{ $labels.namespace }} connection usage is high, over 80% for 5m.
  summary: PolarDB-X DN Connection Usage is High

alert: PolarDB-X DN Connection Usage is Too High
expr: sum by(pod, polardbx_name, namespace) (100 * mysql_global_status_threads_connected / mysql_global_variables_max_connections{polardbx_role="dn"}) >= 90
for: 5m
labels:
  severity: critical
annotations:
  description: PolarDB-X {{ $labels.polardbx_name}} pod {{ $labels.pod }} namespace {{ $labels.namespace }} connection usage is too high, over 90% for 5m.
  summary: PolarDB-X DN Connection Usage is too High

alert: PolarDB-X DN SQL Running Connection is High
expr: sum by(pod, polardbx_name, namespace) (mysql_global_status_threads_running) >= 300
for: 5m
labels:
  severity: warning
annotations:
  description: PolarDB-X {{ $labels.polardbx_name}} pod {{ $labels.pod }} namespace {{ $labels.namespace }} running count is high, over 300.
  summary: PolarDB-X DN SQL Running Connection is High

alert: PolarDB-X DN SQL Running Connection is Too High
expr: sum by(pod, polardbx_name, namespace) (mysql_global_status_threads_running) >= 500
for: 5m
labels:
  severity: critical
annotations:
  description: PolarDB-X {{ $labels.polardbx_name}} pod {{ $labels.pod }} namespace {{ $labels.namespace }} running count is too high, over 500.
  summary: PolarDB-X DN SQL Running Connection is Too High

alert: PolarDB-X DN Slow SQL Count is High
expr: sum by(pod, polardbx_name, namespace) (mysql_global_status_slow_queries) >= 30
for: 5m
labels:
  severity: warning
annotations:
  description: PolarDB-X {{ $labels.polardbx_name}} pod {{ $labels.pod }} namespace {{ $labels.namespace }} running count is high, over 30.
  summary: PolarDB-X DN Slow SQL Count is High

alert: PolarDB-X DN Slow SQL Count is Too High
expr: sum by(pod, polardbx_name, namespace) (mysql_global_status_slow_queries) >= 50
for: 5m
labels:
  severity: critical
annotations:
  description: PolarDB-X {{ $labels.polardbx_name}} pod {{ $labels.pod }} namespace {{ $labels.namespace }} running count is too high, over 50.
  summary: PolarDB-X DN Slow SQL Count is Too High

alert: PolarDB-X Node CPU Usage is High
expr: sum by(namespace, pod) (polardbx_container_cpu_usage_seconds_total:sum_rate{container="engine"}) / sum by(namespace, pod) (kube_pod_container_resource_limits{resource="cpu",unit="core"}) * 100 >= 80
for: 5m
labels:
  severity: warning
annotations:
  description: Namespace {{ $labels.namespace }} pod {{ $labels.pod }} CPU is High, over 80% for 5m.
  summary: PolarDB-X Node CPU is High

alert: PolarDB-X Node CPU Usage is Too High
expr: sum by(namespace, pod) (polardbx_container_cpu_usage_seconds_total:sum_rate{container="engine"}) / sum by(namespace, pod) (kube_pod_container_resource_limits{resource="cpu",unit="core"}) * 100 >= 90
for: 5m
labels:
  severity: critical
annotations:
  description: Namespace {{ $labels.namespace }} pod {{ $labels.pod }} CPU is too High, over 90% for 5m.
  summary: PolarDB-X Node CPU is Too High

alert: PolarDB-X Node Disk Usage is High
expr: sum by(instance, mountpoint) ((node_filesystem_size_bytes{fstype=~"ext.*|xfs|nfs",mountpoint!="/boot",mountpoint!~".*pod.*"} - node_filesystem_free_bytes{fstype=~"ext.*|xfs|nfs",mountpoint!="/boot",mountpoint!~".*pod.*"}) / (node_filesystem_avail_bytes{fstype=~"ext.*|xfs|nfs",mountpoint!="/boot",mountpoint!~".*pod.*"} + (node_filesystem_size_bytes{fstype=~"ext.*|xfs|nfs",mountpoint!="/boot",mountpoint!~".*pod.*"} - node_filesystem_free_bytes{fstype=~"ext.*|xfs|nfs",mountpoint!="/boot",mountpoint!~".*pod.*"}))) * 100 >= 80
for: 5m
labels:
  severity: warning
annotations:
  description: Instance {{ $labels.instance }} mountpoint {{ $labels.mountpoint }} Disk Usage is high, over 80% for 5m.
  summary: PolarDB-X-Node Disk Usage is High

alert: PolarDB-X Node Disk Usage is Too High
expr: sum by(instance, mountpoint) ((node_filesystem_size_bytes{fstype=~"ext.*|xfs|nfs",mountpoint!="/boot",mountpoint!~".*pod.*"} - node_filesystem_free_bytes{fstype=~"ext.*|xfs|nfs",mountpoint!="/boot",mountpoint!~".*pod.*"}) / (node_filesystem_avail_bytes{fstype=~"ext.*|xfs|nfs",mountpoint!="/boot",mountpoint!~".*pod.*"} + (node_filesystem_size_bytes{fstype=~"ext.*|xfs|nfs",mountpoint!="/boot",mountpoint!~".*pod.*"} - node_filesystem_free_bytes{fstype=~"ext.*|xfs|nfs",mountpoint!="/boot",mountpoint!~".*pod.*"}))) * 100 >= 90
for: 5m
labels:
  severity: critical
annotations:
  description: Instance {{ $labels.instance }} mountpoint {{ $labels.mountpoint }} Disk Usage is too high, over 90% for 5m.
  summary: PolarDB-X-Node Disk Usage is Too High

alert: PolarDB-X Node Used Memory is High
expr: sum by(namespace, pod) (container_memory_working_set_bytes{container="engine",id=~"^/kubepods.*"}) / sum by(namespace, pod) (kube_pod_container_resource_limits{container="engine",resource="memory",unit="byte"}) * 100 >= 90
for: 5m
labels:
  severity: warning
annotations:
  description: Namespace {{ $labels.namespace }} pod {{ $labels.pod }} Memory is too high, over 90% for 5m.
  summary: PolarDB-X Node Memory is High

alert: PolarDB-X Node Used Memory is Too High
expr: sum by(namespace, pod) (container_memory_working_set_bytes{container="engine",id=~"^/kubepods.*"}) / sum by(namespace, pod) (kube_pod_container_resource_limits{container="engine",resource="memory",unit="byte"}) * 100 >= 95
for: 5m
labels:
  severity: critical
annotations:
  description: Namespace {{ $labels.namespace }} pod {{ $labels.pod }} Memory is too high, over 95% for 5m.
  summary: PolarDB-X Node Memory is Too High

alert: PolarDB-X-Node IO Usage is High
expr: sum by(device, instance) (rate(node_disk_io_time_seconds_total[5m])) >= 90
for: 5m
labels:
  severity: warning
annotations:
  description: instance {{ $labels.instance }} device {{ $labels.device }} IO Usage is high, over 80% for 5m.
  summary: PolarDB-X-Node IO Usage is High

alert: PolarDB-X-Node IO Usage is Too High
expr: sum by(device, instance) (rate(node_disk_io_time_seconds_total[5m])) >= 90
for: 5m
labels:
  severity: critical
annotations:
  description: instance {{ $labels.instance }} device {{ $labels.device }} IO Usage is too high, over 90% for 5m.
  summary: PolarDB-X-Node IO Usage is Too High