feat(monitoring): add more rules, run grafana in retiolum

.versions/retiolum.json

@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/krebs/retiolum",
-  "rev": "a643fe16365118d53ee2aa2d6205d0f0dae3719f",
-  "date": "2021-09-06T21:02:13+02:00",
-  "path": "/nix/store/r7fn6j86hn9089ls0rgbsp51qsyzmpib-retiolum",
-  "sha256": "1hbdli97ad0ajcm9gj1jnn7y6g0058i3hpdj8hyv2vkxsnshqdw4",
+  "rev": "5384cff550036aa3581f8c6f851392329f28ffe8",
+  "date": "2021-09-19T09:32:14+02:00",
+  "path": "/nix/store/izzhb6p76h96zmlz5xdxa1xrpixhbk4s-retiolum",
+  "sha256": "07318jnkb8zycyi43fydrmrsrbq9ajhb2nnaahj1bxg77pb4wj7a",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false

configs/cloud.nix

@@ -40,11 +40,11 @@
       folders =
         let cloud-dir = "${config.users.users.me.home}/cloud";
         in {
-          "${cloud-dir}/syncthing/common".devices = [ "wilde" "manakish" ];
-          "${cloud-dir}/syncthing/library".devices = [ "wilde" "manakish" "heym" ];
-          "${cloud-dir}/syncthing/mundoiu".devices = [ "wilde" "manakish" "heym" ];
+          "${cloud-dir}/syncthing/common".devices = [ "kabsa" "manakish" ];
+          "${cloud-dir}/syncthing/library".devices = [ "kabsa" "manakish" "heym" ];
+          "${cloud-dir}/syncthing/mundoiu".devices = [ "kabsa" "manakish" "heym" ];
           "${cloud-dir}/syncthing/music" = {
-            devices = [ "wilde" "manakish" "heym" "zaatar" ];
+            devices = [ "kabsa" "manakish" "heym" "zaatar" ];
             id = "music";
           };
         };

configs/mail/fetcher.nix

@@ -47,7 +47,7 @@ in
 
   systemd.services.mail-sync =
   let
-    hosts = [ "manakish.r" "wilde.r" ];
+    hosts = [ "manakish.r" "kabsa.r" ];
   in {
     enable = true;
     wants = [ "network-online.target" config.krebs.secret.files.email-ssh.service ];

configs/monitoring/pull.nix

@@ -5,14 +5,12 @@ in
 {
   services.grafana = {
     enable = true;
-    domain = "monitoring.xn--kiern-0qa.de";
-    port = 2342;
+    domain = "grafana.kmein.r";
+    port = 9444;
     addr = "127.0.0.1";
   };
 
   services.nginx.virtualHosts.${config.services.grafana.domain} = {
-    enableACME = true;
-    forceSSL = true;
     locations."/" = {
       proxyPass = "http://127.0.0.1:${toString config.services.grafana.port}";
       proxyWebsockets = true;
@@ -25,32 +23,56 @@ in
       rules = [
         {
           alert = "ServiceDown";
-          for = "5m";
           expr = ''node_systemd_unit_state{state="failed"} == 1'';
-          labels.severity = "warning";
           annotations = {
-            summary = "{{ $labels.name }} is down.";
+            summary = "{{$labels.job}}: Service {{$labels.name}} failed to start.";
           };
         }
         {
           alert = "RootPartitionFull";
-          for = "30m";
-          expr = ''(node_filesystem_avail_bytes{mountpoint="/"} * 100) / node_filesystem_size_bytes{mountpoint="/"} < ${toString diskFreeThreshold}'';
-          labels.severity = "warning";
+          for = "10m";
+          expr = ''(node_filesystem_free_bytes{mountpoint="/"} * 100) / node_filesystem_size_bytes{mountpoint="/"} < ${toString diskFreeThreshold}'';
           annotations = {
-            summary = "{{ $labels.job }} root disk full.";
+            summary = "{{ $labels.job }}: Filesystem is running out of space soon.";
             description = ''The root disk of {{ $labels.job }} has {{ $value | printf "%.2f" }}% free disk space (threshold at ${toString diskFreeThreshold}%).'';
           };
         }
+        {
+          alert = "RootPartitionFullWeek";
+          for = "1h";
+          expr = ''node_filesystem_free_bytes{mountpoint="/"} ''
+            + ''and predict_linear(node_filesystem_free_bytes{mountpoint="/"}[2d], 7*24*3600) <= 0'';
+          annotations = {
+            summary = "{{$labels.job}}: Filesystem is running out of space in 7 days.";
+          };
+        }
+        {
+          alert = "HighLoad";
+          expr = ''node_load15 / on(job) count(node_cpu_seconds_total{mode="system"}) by (job) >= 1.0'';
+          for = "10m";
+          annotations = {
+            summary = "{{$labels.job}}: Running on high load: {{$value}}";
+          };
+        }
+        {
+          alert = "HighRAM";
+          expr = "node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes < node_memory_MemTotal_bytes * 0.1";
+          for = "1h";
+          annotations.summary = "{{$labels.job}}: Using lots of RAM.";
+        }
         {
           alert = "HostDown";
           expr = ''up == 0'';
           for = "5m";
-          labels.severity = "warning";
           annotations = {
             summary = "Host {{ $labels.job }} down for 5 minutes.";
           };
         }
+        {
+          alert = "Reboot";
+          expr = "time() - node_boot_time_seconds < 300";
+          annotations.summary = "{{$labels.alias}}: Reboot";
+        }
       ];
     }];
   })];
@@ -83,7 +105,20 @@ in
         --alertmanager.url=http://localhost:9093 --log.level=info \
         --store=bolt --bolt.path=/var/lib/alertbot/bot.db \
         --listen.addr="0.0.0.0:16320" \
-        --template.paths=${./template.tmpl}'';
+        --template.paths=${pkgs.writeText "template.tmpl" ''
+          {{ define "telegram.default" }}
+          {{range .Alerts -}}
+          {{ if eq .Status "firing" }}
+            ⚠ <b>{{ index .Annotations "summary"}}</b>
+            {{ index .Annotations "description" }}
+
+            See on <a href="${config.services.grafana.domain}/d/alpUteInz/niveum">Grafana</a>.
+          {{ else -}}
+            😌 <del>{{ index .Annotations "summary"}}</del>
+          {{- end }}
+          {{end -}}
+          {{end}}
+        ''}'';
     };
   };
 
@@ -126,7 +161,9 @@ in
     }
   ];
 
-  networking.firewall.allowedTCPPorts = [ lokiConfig.server.http_listen_port ];
+  networking.firewall.allowedTCPPorts = [
+    lokiConfig.server.http_listen_port
+  ];
 
   services.loki = {
     enable = true;

configs/monitoring/template.tmpl

@@ -1,25 +0,0 @@
-{{ define "telegram.default" }}
-{{range .Alerts -}}
-{{ $severity := index .Labels "severity" }}
-{{ $desc := "" }}
-{{ $grafana := "d/alpUteInz/niveum" }}
-{{ if eq .Status "firing" }}
-  {{ $desc = index .Annotations "description" }}
-  {{ $grafana = index .Annotations "url" }}
-  {{- if eq $severity "critical" -}}
-    🔥 <b>CRITICAL</b> 🔥
-  {{- else if eq $severity "warning" -}}
-    ⚠ <b>WARNING</b> ⚠
-  {{- else -}}
-    <b>{{ $severity }}</b>
-  {{- end -}}
-{{ else -}}
-  {{ $desc = "The issue has been resolved. 😌" }}
-  🎉 <del>RESOLVED</del> 🎉
-{{- end }}
-<i>{{ index .Labels "alertname"}}</i>
-{{ index .Annotations "summary"}}: {{ $desc }}
-
-See on <a href="https://monitoring.xn--kiern-0qa.de/{{ $grafana }}">Grafana</a>.
-{{end -}}
-{{end}}

configs/ssh.nix

@@ -35,8 +35,8 @@ in {
         user = "root";
         port = sshPort;
       };
-      wilde = {
-        hostname = "wilde.r";
+      kabsa = {
+        hostname = "kabsa.r";
         user = "kfm";
         port = sshPort;
       };

configs/tuna.nix

@@ -23,10 +23,10 @@ in
       cert = toString <system-secrets/syncthing/cert.pem>;
       key = toString <system-secrets/syncthing/key.pem>;
       devices = {
-        inherit ((import <niveum/lib>).syncthing.devices) wilde manakish heym;
+        inherit ((import <niveum/lib>).syncthing.devices) kabsa manakish heym;
       };
       folders.${config.services.mpd.musicDirectory} = {
-        devices = [ "heym" "wilde" "manakish" ];
+        devices = [ "heym" "kabsa" "manakish" ];
         id = "music";
         type = "receiveonly";
       };
@@ -103,11 +103,8 @@ in
     recommendedOptimisation = true;
     recommendedProxySettings = true;
     recommendedTlsSettings = true;
-    virtualHosts.default = {
+    virtualHosts."radio.kmein.r" = {
       basicAuth.dj = password;
-      locations."= /listen.ogg" = {
-        proxyPass = "http://127.0.0.1:${toString multi-room-audio-port}";
-      };
       locations."/" = {
         proxyPass = "http://127.0.0.1:${toString config.services.tuna.webPort}";
         proxyWebsockets = true;

deploy.nix

@@ -43,10 +43,10 @@ in {
     name = "zaatar";
     address = "zaatar.r";
   });
-  wilde = writeDeploy "deploy-wilde" (regularSystem {
-    path = systems/wilde;
-    name = "wilde";
-    address = "wilde.r";
+  kabsa = writeDeploy "deploy-kabsa" (regularSystem {
+    path = systems/kabsa;
+    name = "kabsa";
+    address = "kabsa.r";
   });
   toum = writeDeploy "deploy-toum" (regularSystem {
     path = systems/toum;

lib/default.nix

@@ -43,7 +43,7 @@ rec {
   };
 
   syncthing.devices = {
-    wilde.id = "R6DEBD7-G5RYDKN-VFA3HPO-WX4DNVI-373F7OQ-AW5MZTT-3L4BDVW-Y6ROEAF";
+    kabsa.id = "R6DEBD7-G5RYDKN-VFA3HPO-WX4DNVI-373F7OQ-AW5MZTT-3L4BDVW-Y6ROEAF";
     heym.id = "HLQSG3D-WSKLA6S-MEYQ3EU-GDBGABE-PY53RQ6-SWQAP2I-Z5MVBVX-MYPJXAM";
     manakish.id = "AJVBWR2-VFFAGZF-7ZF5JAX-T63GMOG-NZ446WK-MC5E6WK-6X6Q2HE-QQA2JQ3";
     toum.id = "CBJQXFF-FMFGWFU-2J6FMPR-SRDTSGX-7NHOYOH-CQCABKF-KQJMRJC-SDE24Q4";

lib/local-network.nix

@@ -1,7 +1,7 @@
 {
   toum = "192.168.178.24";
   zaatar = "192.168.178.21";
-  wilde = "192.168.178.32";
+  kabsa = "192.168.178.32";
   android = "192.168.178.35";
   manakish = "192.168.178.29";
 

systems/kabsa/configuration.nix

@@ -35,13 +35,13 @@ in
 
   boot.loader.efi.canTouchEfiVariables = true;
 
-  networking.hostName = "wilde";
+  networking.hostName = "kabsa";
 
   networking.wireless.interfaces = [ "wlp3s0" ];
 
   networking.retiolum = {
     ipv4 = "10.243.2.4";
-    ipv6 = "42:0:3c46:907c:1fb8:b74f:c59b:1ee3";
+    ipv6 = "42:0:3c46:861f:a118:8e9a:82c9:3d";
   };
 
   system.stateVersion = "19.03";