feat(monitoring): streamline alerting

This reverts commit 630d99e191.

configs/hu-berlin.nix

@@ -67,7 +67,7 @@ in {
         host = forti-ssl.vpn.hu-berlin.de
         port = 443
         trusted-cert = 42193a913d276d9eb86217612956e1e6464d6f07bed5393a4787c87adc4bd359
-        username = ${eduroam.identity}
+        username = ${eduroam.identity}@split_tunnel
         password = ${eduroam.password}
       ''}
     '';

lib/i3status-rust.nix

@@ -121,22 +121,16 @@ in {
         print("🌅 {} 🌇 {} {} {}".format(sun["sunrise"].strftime("%R"), sun["sunset"].strftime("%R"), moon_phases[closest_phase], round(current_phase, 1)))
       '';
     }
-    (let service = "openvpn-hu-berlin"; in {
+    {
       block = "custom";
       interval = 5;
-      command = pkgs.writers.writeDash "net-device" ''
+      command = pkgs.writers.writeDash "hu-berlin-vpn" ''
         PATH=${lib.makeBinPath [ pkgs.systemd ]}
-        systemctl is-active --quiet ${service}.service && echo "🎓👍" || echo "🎓👎"
+        (systemctl is-active --quiet openvpn-hu-berlin.service && echo "🎓👍 (OpenVPN)") \
+          || (systemctl is-active --quiet hu-vpn.service && echo "🎓👍 (PPP+SSL)") \
+          || echo "🎓👎"
       '';
-      on_click = pkgs.writers.writeDash "toggle" ''
-        PATH=${lib.makeBinPath [ pkgs.systemd pkgs.libnotify ]}
-        systemctl is-active --quiet ${service}.service && {
-          systemctl stop ${service}.service && notify-send -a "${service}" stopped
-        } || {
-          systemctl start ${service}.service && notify-send -a "${service}" started
-        }
-      '';
-    })
+    }
     {
       block = "net";
       device = wirelessInterface;

systems/makanek/monitoring/blackbox.nix

@@ -3,7 +3,7 @@
 {
   modules.http_2xx = {
     http = {
-      fail_if_not_ssl = true;
+      fail_if_not_ssl = false;
       ip_protocol_fallback = false;
       method = "GET";
       no_follow_redirects = false;

systems/makanek/monitoring/default.nix

@@ -114,24 +114,27 @@ in
     }];
   })];
 
-  systemd.services.alertmanager-irc = {
+  systemd.services.alertmanager-bot-telegram = {
     wantedBy = [ "multi-user.target" ];
     after = [ "ip-up.target" ];
+    environment.TELEGRAM_ADMIN = "18980945";
+    environment.TELEGRAM_TOKEN = lib.strings.fileContents <system-secrets/telegram/prometheus.token>;
     serviceConfig = {
+      Restart = "on-failure";
+      RestartSec = "15s";
       DynamicUser = true;
-      StateDirectory = "alert-irc";
-      ExecStart = ''${pkgs.alertmanager-irc-relay}/bin/alertmanager-irc-relay \
-        --config ${(pkgs.formats.yaml {}).generate "config.yaml" {
-          http_host = "0.0.0.0";
-          http_port = 16330;
-          irc_host = "irc.r";
-          irc_port = 6667;
-          irc_use_ssl = false;
-          irc_nickname = "niveum";
-          irc_channels = [ { name = "#niveum"; } ];
-          msg_template = ''{{ index .Annotations "summary" }} ({{ .Status }})'';
-        }}
-      '';
+      StateDirectory = "alertbot";
+      ExecStart = ''${pkgs.alertmanager-bot-telegram}/bin/alertmanager-bot \
+        --alertmanager.url=http://localhost:9093 --log.level=info \
+        --store=bolt --bolt.path=/var/lib/alertbot/bot.db \
+        --listen.addr="0.0.0.0:16320" \
+        --template.paths=${pkgs.writeText "template.tmpl" ''
+          {{ define "telegram.default" }}
+          {{range .Alerts -}}
+          <b>{{.Status}}</b> {{ index .Annotations "summary"}}
+          {{end -}}
+          {{end}}
+        ''}'';
     };
   };
 
@@ -146,16 +149,20 @@ in
       };
       receivers = [{
         name = "me";
-        webhook_configs = [
-          {
-            url = "http://localhost:16330/niveum";
-            send_resolved = true;
-          }
-        ];
+        webhook_configs = [{
+          url = "http://localhost:16320";
+          send_resolved = true;
+        }];
       }];
     };
   };
 
+  services.prometheus.alertmanagers = [{
+    scheme = "http";
+    path_prefix = "/";
+    static_configs = [ { targets = [ "localhost:9093" ]; } ];
+  }];
+
   services.prometheus.scrapeConfigs = [
     {
       job_name = "makanek";

systems/makanek/weechat.nix

@@ -33,9 +33,6 @@ in {
             look.mouse = true;
             look.prefix_align_max = 15;
             color.chat_nick_colors = lib.lists.subtractLists (lib.range 52 69 ++ lib.range 231 248) (lib.range 31 254);
-            look.highlight_tags = [
-              "irc_notice+nick_niveum"
-            ];
           };
           irc = {
             look = {
@@ -64,7 +61,7 @@ in {
                 autoconnect = true;
                 address = "irc.libera.chat/6697";
                 ssl = true;
-                autojoin = [ "#flipdot" "#haskell" "#nixos" "#fysi" "#binaergewitter" "#neovim" "#niveum" "#lojban" "#vim" ];
+                autojoin = [ "#flipdot" "#haskell" "#nixos" "#fysi" "#binaergewitter" "#neovim" "#lojban" "#vim" ];
                 sasl_mechanism = "plain";
                 sasl_username = nick;
                 sasl_password = lib.strings.fileContents <system-secrets/irc/libera>;
@@ -83,7 +80,7 @@ in {
               retiolum = {
                 autoconnect = true;
                 address = "irc.r";
-                autojoin = [ "#xxx" "#brockman" "#flix" "#niveum" ];
+                autojoin = [ "#xxx" "#brockman" "#flix" "#autowifi" ];
                 command = lib.concatStringsSep "\\;" [
                   "/oper admin aidsballs"
                   "/msg nickserv always-on true"