alertmanager: alert lassulus for mastodon outtime

systems/makanek/monitoring/default.nix

@@ -3,11 +3,13 @@
   config,
   pkgs,
   ...
-}: let
+}:
+let
   lokiConfig = import ./loki.nix;
   blackboxConfig = import ./blackbox.nix;
   inherit (import ../../../lib) restic;
-in {
+in
+{
   services.grafana = {
     enable = true;
     settings = {
@@ -80,9 +82,11 @@ in {
     }
   ];
 
-  services.prometheus.rules = let
+  services.prometheus.rules =
+    let
       diskFreeThreshold = 10;
-  in [
+    in
+    [
       (builtins.toJSON {
         groups = [
           {
@@ -181,6 +185,12 @@ in {
                 expr = "time() - node_boot_time_seconds < 300";
                 annotations.description = "{{$labels.instance}} rebooted";
               }
+              {
+                alert = "Mastodon";
+                expr = ''probe_success{instance="https://social.krebsco.de"}'';
+                for = "5m";
+                annotations.description = "Mastodon instance {{$labels.instance}} is down";
+              }
               {
                 alert = "ProbeFailed";
                 expr = "probe_success == 0";
@@ -217,7 +227,6 @@ in {
       })
     ];
 
-
   # ref https://github.com/Mic92/dotfiles/blob/f44bac5dd6970ed3fbb4feb906917331ec3c2be5/machines/eva/modules/prometheus/default.nix
   systemd.services.matrix-hook = {
     description = "Matrix Hook";
@@ -246,6 +255,33 @@ in {
     };
   };
 
+  systemd.services.matrix-hook-lassulus = {
+    description = "Matrix Hook";
+    after = [ "network.target" ];
+    wantedBy = [ "multi-user.target" ];
+    environment = {
+      HTTP_ADDRESS = "[::1]";
+      HTTP_PORT = "9089";
+      MX_HOMESERVER = "https://matrix.4d2.org";
+      MX_ID = "@lakai:4d2.org";
+      MX_ROOMID = "!MJAGqBAOKZGMywzwkI:lassul.us";
+      MX_MSG_TEMPLATE = "${pkgs.matrix-hook}/message.html.tmpl";
+    };
+    serviceConfig = {
+      EnvironmentFile = [
+        # format: MX_TOKEN=<token>
+        config.age.secrets.matrix-token-lakai-env.path
+      ];
+      Type = "simple";
+      ExecStart = "${pkgs.matrix-hook}/bin/matrix-hook";
+      Restart = "always";
+      RestartSec = "10";
+      DynamicUser = true;
+      User = "matrix-hook";
+      Group = "matrix-hook";
+    };
+  };
+
   age.secrets = {
     matrix-token-lakai-env.file = ../../../secrets/matrix-token-lakai-env.age;
   };
@@ -260,8 +296,23 @@ in {
         group_wait = "30s";
         repeat_interval = "24h";
         receiver = "matrix";
+        routes = [
+          {
+            receiver = "lassulus";
+            matchers = [ "alertname = \"Mastodon\"" ];
+          }
+        ];
       };
       receivers = [
+        {
+          name = "lassulus";
+          webhook_configs = [
+            {
+              url = "http://localhost:9089/alert";
+              max_alerts = 5;
+            }
+          ];
+        }
         {
           name = "matrix";
           webhook_configs = [