kfm/niveum
1
0
Fork 0
mirror of https://github.com/kmein/niveum synced 2026-03-20 03:51:07 +01:00
Code Releases Activity

alertmanager: summary -> description

Browse Source
This commit is contained in:
Kierán Meinhardt
2024-11-09 16:55:14 +01:00
parent f3a5e0a54c
commit 63f5db1c58
1 changed files with 19 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
systems/makanek/monitoring/default.nix
View File

@@ -92,7 +92,7 @@ in {
alert = "HostSystemdServiceCrashed"; alert = "HostSystemdServiceCrashed";
expr = ''(node_systemd_unit_state{state="failed"} == 1) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}''; expr = ''(node_systemd_unit_state{state="failed"} == 1) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}'';
annotations = { annotations = {
summary = "{{$labels.name}} failed on {{$labels.instance}}"; description = "{{$labels.name}} failed on {{$labels.instance}}";
}; };
} }
{ {
@@ -100,7 +100,7 @@ in {
for = "10m"; for = "10m";
expr = ''(node_filesystem_free_bytes{mountpoint="/"} * 100) / node_filesystem_size_bytes{mountpoint="/"} < ${toString diskFreeThreshold}''; expr = ''(node_filesystem_free_bytes{mountpoint="/"} * 100) / node_filesystem_size_bytes{mountpoint="/"} < ${toString diskFreeThreshold}'';
annotations = { annotations = {
summary = ''{{ $labels.instance }} running out of space: {{ $value | printf "%.2f" }}% < ${toString diskFreeThreshold}%''; description = ''{{ $labels.instance }} running out of space: {{ $value | printf "%.2f" }}% < ${toString diskFreeThreshold}%'';
}; };
} }
{ {
@@ -110,7 +110,7 @@ in {
''node_filesystem_free_bytes{mountpoint="/"} '' ''node_filesystem_free_bytes{mountpoint="/"} ''
+ ''and predict_linear(node_filesystem_free_bytes{mountpoint="/"}[2d], 7*24*3600) <= 0''; + ''and predict_linear(node_filesystem_free_bytes{mountpoint="/"}[2d], 7*24*3600) <= 0'';
annotations = { annotations = {
summary = "{{$labels.instance}} running out of space in 7 days"; description = "{{$labels.instance}} running out of space in 7 days";
}; };
} }
{ {
@@ -118,98 +118,98 @@ in {
expr = ''node_load15 / on(job) count(node_cpu_seconds_total{mode="system"}) by (job) >= 1.0''; expr = ''node_load15 / on(job) count(node_cpu_seconds_total{mode="system"}) by (job) >= 1.0'';
for = "10m"; for = "10m";
annotations = { annotations = {
summary = "{{$labels.instance}} running on high load: {{$value}}"; description = "{{$labels.instance}} running on high load: {{$value}}";
}; };
} }
{ {
alert = "HostUnusualNetworkThroughputIn"; alert = "HostUnusualNetworkThroughputIn";
expr = ''(rate(node_network_receive_bytes_total[2m])) / 1024 / 1024 > 100''; expr = ''(rate(node_network_receive_bytes_total[2m])) / 1024 / 1024 > 100'';
for = "5m"; for = "5m";
annotations.summary = "Host unusual network throughput in (instance {{ $labels.instance }})"; annotations.description = "Host unusual network throughput in (instance {{ $labels.instance }})";
} }
{ {
alert = "HostUnusualNetworkThroughputOut"; alert = "HostUnusualNetworkThroughputOut";
expr = ''(rate(node_network_transmit_bytes_total[2m])) / 1024 / 1024 > 100''; expr = ''(rate(node_network_transmit_bytes_total[2m])) / 1024 / 1024 > 100'';
for = "5m"; for = "5m";
annotations.summary = "Host unusual network throughput out (instance {{ $labels.instance }})"; annotations.description = "Host unusual network throughput out (instance {{ $labels.instance }})";
} }
{ {
alert = "HostUnusualDiskReadRate"; alert = "HostUnusualDiskReadRate";
expr = ''(rate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50''; expr = ''(rate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50'';
for = "5m"; for = "5m";
annotations.summary = "Host unusual disk read rate (instance {{ $labels.instance }})"; annotations.description = "Host unusual disk read rate (instance {{ $labels.instance }})";
} }
{ {
alert = "HostUnusualDiskWriteRate"; alert = "HostUnusualDiskWriteRate";
expr = ''(rate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50''; expr = ''(rate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50'';
for = "2m"; for = "2m";
annotations.summary = "Host unusual disk write rate (instance {{ $labels.instance }})"; annotations.description = "Host unusual disk write rate (instance {{ $labels.instance }})";
} }
{ {
alert = "HostOutOfInodes"; alert = "HostOutOfInodes";
expr = ''node_filesystem_files_free{fstype!="msdosfs"} / node_filesystem_files{fstype!="msdosfs"} * 100 < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0''; expr = ''node_filesystem_files_free{fstype!="msdosfs"} / node_filesystem_files{fstype!="msdosfs"} * 100 < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0'';
for = "2m"; for = "2m";
annotations.summary = "Host out of inodes (instance {{ $labels.instance }})"; annotations.description = "Host out of inodes (instance {{ $labels.instance }})";
} }
{ {
alert = "HostInodesWillFillIn24Hours"; alert = "HostInodesWillFillIn24Hours";
expr = ''node_filesystem_files_free{fstype!="msdosfs"} / node_filesystem_files{fstype!="msdosfs"} * 100 < 10 and predict_linear(node_filesystem_files_free{fstype!="msdosfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly{fstype!="msdosfs"} == 0''; expr = ''node_filesystem_files_free{fstype!="msdosfs"} / node_filesystem_files{fstype!="msdosfs"} * 100 < 10 and predict_linear(node_filesystem_files_free{fstype!="msdosfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly{fstype!="msdosfs"} == 0'';
for = "2m"; for = "2m";
annotations.summary = "Host inodes will fill in 24 hours (instance {{ $labels.instance }})"; annotations.description = "Host inodes will fill in 24 hours (instance {{ $labels.instance }})";
} }
{ {
alert = "HighRAM"; alert = "HighRAM";
expr = "node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes < node_memory_MemTotal_bytes * 0.1"; expr = "node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes < node_memory_MemTotal_bytes * 0.1";
for = "1h"; for = "1h";
annotations.summary = "{{$labels.instance}} using lots of RAM"; annotations.description = "{{$labels.instance}} using lots of RAM";
} }
{ {
alert = "UptimeMonster"; alert = "UptimeMonster";
expr = "time() - node_boot_time_seconds > 2592000"; expr = "time() - node_boot_time_seconds > 2592000";
annotations.summary = "uptime monster {{$labels.instance}} up for more than 30 days"; annotations.description = "uptime monster {{$labels.instance}} up for more than 30 days";
} }
{ {
alert = "HostDown"; alert = "HostDown";
expr = ''up == 0''; expr = ''up == 0'';
for = "5m"; for = "5m";
annotations = { annotations = {
summary = "{{ $labels.instance }} seeming down since 5 minutes"; description = "{{ $labels.instance }} seeming down since 5 minutes";
}; };
} }
{ {
alert = "Reboot"; alert = "Reboot";
expr = "time() - node_boot_time_seconds < 300"; expr = "time() - node_boot_time_seconds < 300";
annotations.summary = "{{$labels.instance}} rebooted"; annotations.description = "{{$labels.instance}} rebooted";
} }
{ {
alert = "ProbeFailed"; alert = "ProbeFailed";
expr = "probe_success == 0"; expr = "probe_success == 0";
for = "5m"; for = "5m";
annotations.summary = "HTTP probe failed for {{$labels.instance}}"; annotations.description = "HTTP probe failed for {{$labels.instance}}";
} }
{ {
alert = "SlowProbe"; alert = "SlowProbe";
expr = "avg_over_time(probe_http_duration_seconds[1m]) > 1"; expr = "avg_over_time(probe_http_duration_seconds[1m]) > 1";
for = "5m"; for = "5m";
annotations.summary = "HTTP probe slow for {{$labels.instance}}"; annotations.description = "HTTP probe slow for {{$labels.instance}}";
} }
{ {
alert = "HttpStatusCode"; alert = "HttpStatusCode";
expr = "probe_http_status_code != 0 AND (probe_http_status_code <= 199 OR probe_http_status_code >= 400)"; expr = "probe_http_status_code != 0 AND (probe_http_status_code <= 199 OR probe_http_status_code >= 400)";
for = "5m"; for = "5m";
annotations.summary = "status code {{$value}} for {{$labels.instance}}"; annotations.description = "status code {{$value}} for {{$labels.instance}}";
} }
{ {
alert = "SslExpirySoon"; alert = "SslExpirySoon";
expr = "probe_ssl_earliest_cert_expiry - time() < 86400 * 30"; expr = "probe_ssl_earliest_cert_expiry - time() < 86400 * 30";
for = "5m"; for = "5m";
annotations.summary = "SSL certificate for {{$labels.instance}} expires in 30 days"; annotations.description = "SSL certificate for {{$labels.instance}} expires in 30 days";
} }
{ {
alert = "SslExpiry"; alert = "SslExpiry";
expr = "probe_ssl_earliest_cert_expiry - time() <= 0"; expr = "probe_ssl_earliest_cert_expiry - time() <= 0";
for = "5m"; for = "5m";
annotations.summary = "SSL certificate for {{$labels.instance}} has expired"; annotations.description = "SSL certificate for {{$labels.instance}} has expired";
} }
]; ];
} }