flake.lock: Update

Flake lock file updates:

• Updated input 'agenix':
    'github:ryantm/agenix/531beac616433bac6f9e2a19feb8e99a22a66baf?narHash=sha256-9P1FziAwl5%2B3edkfFcr5HeGtQUtrSdk/MksX39GieoA%3D' (2025-06-17)
  → 'github:ryantm/agenix/9edb1787864c4f59ae5074ad498b6272b3ec308d?narHash=sha256-NA/FT2hVhKDftbHSwVnoRTFhes62%2B7dxZbxj5Gxvghs%3D' (2025-08-05)
• Updated input 'home-manager':
    'github:nix-community/home-manager/501cfec8277f931a9c9af9f23d3105c537faeafe?narHash=sha256-tWosziZTT039x6PgEZUhzGlV8oLvdDmIgKTE8ESMaEA%3D' (2025-07-02)
  → 'github:nix-community/home-manager/fc3add429f21450359369af74c2375cb34a2d204?narHash=sha256-oV695RvbAE4%2BR9pcsT9shmp6zE/%2BIZe6evHWX63f2Qg%3D' (2025-07-27)
• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/7a732ed41ca0dd64b4b71b563ab9805a80a7d693?narHash=sha256-u7ubvtxdTnFPpV27AHpgoKn7qHuE7sgWgza/1oj5nzA%3D' (2025-07-03)
  → 'github:NixOS/nixpkgs/fc756aa6f5d3e2e5666efcf865d190701fef150a?narHash=sha256-eogqv6FqZXHgqrbZzHnq43GalnRbLTkbBbFtEfm1RSc%3D' (2025-08-08)
• Updated input 'nixpkgs-unstable':
    'github:NixOS/nixpkgs/07c3e0e2be593551abdccacabe29cc57206b396f?narHash=sha256-30JXBxkJD3pxBBGQwl/DDwxxJDGQ1nvkvWwEtTsmhA8%3D' (2025-07-05)
  → 'github:NixOS/nixpkgs/796902fa97f5d4174251562eee6fa4d138c4a5e8?narHash=sha256-BExvuIabyDAgm8oLgGi/rbTbCouydsLMLaL2Fzpdvcc%3D' (2025-08-10)
• Updated input 'nur':
    'github:nix-community/NUR/a26411970baba3604e425f23e5293da492069e4a?narHash=sha256-iDjz1tBd/ibtmZZse4k1NBxfPR8g9K5W4sRgCyOnrj0%3D' (2025-07-05)
  → 'github:nix-community/NUR/27dfc5e3bedd281be925bc290351890571a720d2?narHash=sha256-aLtVkt7vTReGT/M51TvKpUNOhvs%2BXFAdimadlUiOPfg%3D' (2025-08-09)
• Updated input 'nur/nixpkgs':
    'github:nixos/nixpkgs/3016b4b15d13f3089db8a41ef937b13a9e33a8df?narHash=sha256-P/SQmKDu06x8yv7i0s8bvnnuJYkxVGBWLWHaU%2Btt4YY%3D' (2025-06-30)
  → 'github:nixos/nixpkgs/c2ae88e026f9525daf89587f3cbee584b92b6134?narHash=sha256-erbiH2agUTD0Z30xcVSFcDHzkRvkRXOQ3lb887bcVrs%3D' (2025-08-06)
• Removed input 'nur/treefmt-nix'
• Removed input 'nur/treefmt-nix/nixpkgs'
• Updated input 'rust-overlay':
    'github:oxalica/rust-overlay/9e5e62a33a929a67a5427fb7324a6f583dced0b2?narHash=sha256-dYO5X5jK8bpQOeRAo8R5aUt6M/%2BJi1cZgstZI7SQ2IA%3D' (2025-07-05)
  → 'github:oxalica/rust-overlay/ac39ab4c8ed7cefe48d5ae5750f864422df58f01?narHash=sha256-wgVgOsyJUDn2ZRpzu2gELKALoJXlBSoZJSln%2BTlg5Pw%3D' (2025-08-09)
• Updated input 'scripts':
    'github:kmein/scripts/f44c7a4a6caa1ef5d6b7bf7e93acea0d96f30c21?narHash=sha256-mDDxMwKFURX1K1Z8X/kmt%2BjYjswofDf0br%2BMkw2tmSE%3D' (2025-06-02)
  → 'github:kmein/scripts/cc37fa4aec70f53731b9131bb8830b4445b75b3d?narHash=sha256-TnfXFloY4Ntq%2B0hp%2Bq9GGmuhtB1oueFiB%2BpcBUNYzFs%3D' (2025-07-20)
• Updated input 'stockholm':
    'github:krebs/stockholm/d4abc837cc7b87b4f23fe48cc306df26e3de7aab?narHash=sha256-i2s6jU%2B8GLKVjhWDyvFYxmXI7A44c9p6apPPyKt0ETk%3D' (2025-05-13)
  → 'github:krebs/stockholm/fc32e4609140fffa1312a4ca1aeea550b7467448?narHash=sha256-Mo2BkJXIz6HKM8cX2S7bRdX6Q3E1UOcyVL4v10QEUzk%3D' (2025-08-09)
• Updated input 'stockholm/buildbot-nix':
    'github:Mic92/buildbot-nix/7ad9b4886eccb5eecc0686a16266ddabf6cbefe9?narHash=sha256-mRnIdJLP%2B0NSim9ao30ue0Z3ttSuxzXwQG7UN1KuKfU%3D' (2025-05-07)
  → 'github:Mic92/buildbot-nix/47ad4c7afb169df6f9d48d0df3d7e2f71d9ddd8f?narHash=sha256-vCYcc/b8WizF6vnjuRVxSiU8hy9L3vOTWDVKpWM7xRE%3D' (2025-07-03)
• Updated input 'stockholm/buildbot-nix/flake-parts':
    'github:hercules-ci/flake-parts/c621e8422220273271f52058f618c94e405bb0f5?narHash=sha256-hIshGgKZCgWh6AYJpJmRgFdR3WUbkY04o82X05xqQiY%3D' (2025-04-01)
  → 'github:hercules-ci/flake-parts/77826244401ea9de6e3bac47c2db46005e1f30b5?narHash=sha256-Tyw1RjYEsp5scoigs1384gIg6e0GoBVjms4aXFfRssQ%3D' (2025-07-01)
• Updated input 'stockholm/buildbot-nix/hercules-ci-effects':
    'github:hercules-ci/hercules-ci-effects/5b6cec51c9ec095a0d3fd4c8eeb53eb5c59ae33e?narHash=sha256-1Z4WPGVky4w3lrhrgs89OKsLzPdtkbi1bPLNFWsoLfY%3D' (2025-04-15)
  → 'github:hercules-ci/hercules-ci-effects/231726642197817d20310b9d39dd4afb9e899489?narHash=sha256-EaAJhwfJGBncgIV/0NlJviid2DP93cTMc9h0q6P6xXk%3D' (2025-05-23)
• Updated input 'stockholm/buildbot-nix/treefmt-nix':
    'github:numtide/treefmt-nix/29ec5026372e0dec56f890e50dbe4f45930320fd?narHash=sha256-4h3s1L/kKqt3gMDcVfN8/4v2jqHrgLIe4qok4ApH5x4%3D' (2025-05-02)
  → 'github:numtide/treefmt-nix/ac8e6f32e11e9c7f153823abc3ab007f2a65d3e1?narHash=sha256-0IEdQB1nS%2BuViQw4k3VGUXntjkDp7aAlqcxdewb/hAc%3D' (2025-06-26)
• Updated input 'stockholm/nixpkgs':
    'github:NixOS/nixpkgs/d89fc19e405cb2d55ce7cc114356846a0ee5e956?narHash=sha256-3e%2BAVBczosP5dCLQmMoMEogM57gmZ2qrVSrmq9aResQ%3D' (2025-05-10)
  → 'github:NixOS/nixpkgs/1fd8bada0b6117e6c7eb54aad5813023eed37ccb?narHash=sha256-J1kI6oAj25IG4EdVlg2hQz8NZTBNYvIS0l4wpr9KcUo%3D' (2025-07-06)
• Updated input 'stylix':
    'github:danth/stylix/5dd301b72207d4fd8d8b929abd88ba1c486d1744?narHash=sha256-jqRbWjB8aH2qzq6nMQpwkzVBR4o9lNxAHFmRgGwnJ94%3D' (2025-07-04)
  → 'github:danth/stylix/312dec38b2231b21f36903d1bdce96daa11548ff?narHash=sha256-AzAYdZlat002vCjCKWdFpGi2xUaiOU4DtIPnv1nomD8%3D' (2025-08-07)
• Removed input 'stylix/flake-compat'
• Removed input 'stylix/git-hooks'
• Removed input 'stylix/git-hooks/flake-compat'
• Removed input 'stylix/git-hooks/gitignore'
• Removed input 'stylix/git-hooks/gitignore/nixpkgs'
• Removed input 'stylix/git-hooks/nixpkgs'
• Removed input 'stylix/home-manager'

README.md

@@ -1,5 +1,7 @@
 # niveum
 
+> I must Create a System, or be enslav'd by another Man's. —William Blake
+
 > [nĭvĕus](https://logeion.uchicago.edu/niveus), a, um, adj. [nix], _of_ or _from snow, snowy, snow-_ (poet.)
 >
 > 1. Lit.: aggeribus niveis informis, Verg. G. 3, 354: aqua, _cooled with snow_, Mart. 12, 17, 6; cf. id. 14, 104 and 117: mons, _covered with snow_, Cat. 64, 240.—

configs/wallpaper.nix

@@ -17,11 +17,7 @@ in {
       chmod o+rx ${stateDir}
       cd ${stateDir}
       (${pkgs.curl}/bin/curl -s -o wallpaper.tmp -z wallpaper.tmp ${lib.escapeShellArg url} && cp wallpaper.tmp wallpaper) || :
-      if [ -z $SWAYSOCK ]; then
+      ${pkgs.feh}/bin/feh --no-fehbg --bg-scale wallpaper
-        ${pkgs.feh}/bin/feh --no-fehbg --bg-scale wallpaper
-      else
-        ${pkgs.sway}/bin/swaymsg -s $SWAYSOCK 'output * bg ${stateDir}/wallpaper fill'
-      fi
     '';
     startAt = "*:00,10,20,30,40,50";
     serviceConfig = {

flake.lock

@@ -12,11 +12,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1750173260,
+        "lastModified": 1754433428,
-        "narHash": "sha256-9P1FziAwl5+3edkfFcr5HeGtQUtrSdk/MksX39GieoA=",
+        "narHash": "sha256-NA/FT2hVhKDftbHSwVnoRTFhes62+7dxZbxj5Gxvghs=",
         "owner": "ryantm",
         "repo": "agenix",
-        "rev": "531beac616433bac6f9e2a19feb8e99a22a66baf",
+        "rev": "9edb1787864c4f59ae5074ad498b6272b3ec308d",
         "type": "github"
       },
       "original": {
@@ -1045,11 +1045,11 @@
     },
     "nixpkgs-unstable_2": {
       "locked": {
-        "lastModified": 1754182383,
+        "lastModified": 1754787963,
-        "narHash": "sha256-hON5G3ENyUGUpqImXT+Ag7TWbBsfthwqxMcAw8EiKw8=",
+        "narHash": "sha256-BExvuIabyDAgm8oLgGi/rbTbCouydsLMLaL2Fzpdvcc=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "e960e6e1dc689e2b5fe9e3ee824fc12860d9f27c",
+        "rev": "796902fa97f5d4174251562eee6fa4d138c4a5e8",
         "type": "github"
       },
       "original": {
@@ -1167,11 +1167,11 @@
     },
     "nixpkgs_7": {
       "locked": {
-        "lastModified": 1754028485,
+        "lastModified": 1754689972,
-        "narHash": "sha256-IiiXB3BDTi6UqzAZcf2S797hWEPCRZOwyNThJIYhUfk=",
+        "narHash": "sha256-eogqv6FqZXHgqrbZzHnq43GalnRbLTkbBbFtEfm1RSc=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "59e69648d345d6e8fef86158c555730fa12af9de",
+        "rev": "fc756aa6f5d3e2e5666efcf865d190701fef150a",
         "type": "github"
       },
       "original": {
@@ -1183,11 +1183,11 @@
     },
     "nixpkgs_8": {
       "locked": {
-        "lastModified": 1753939845,
+        "lastModified": 1754498491,
-        "narHash": "sha256-K2ViRJfdVGE8tpJejs8Qpvvejks1+A4GQej/lBk5y7I=",
+        "narHash": "sha256-erbiH2agUTD0Z30xcVSFcDHzkRvkRXOQ3lb887bcVrs=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "94def634a20494ee057c76998843c015909d6311",
+        "rev": "c2ae88e026f9525daf89587f3cbee584b92b6134",
         "type": "github"
       },
       "original": {
@@ -1330,11 +1330,11 @@
         "nixpkgs": "nixpkgs_8"
       },
       "locked": {
-        "lastModified": 1754178761,
+        "lastModified": 1754771961,
-        "narHash": "sha256-UhAauJWx4bcopgP29mm7KFVVw38ZRUYp22I6sMKrUMY=",
+        "narHash": "sha256-aLtVkt7vTReGT/M51TvKpUNOhvs+XFAdimadlUiOPfg=",
         "owner": "nix-community",
         "repo": "NUR",
-        "rev": "dbd17f07b63f5fc630ff9d3b5b832636259ac8e8",
+        "rev": "27dfc5e3bedd281be925bc290351890571a720d2",
         "type": "github"
       },
       "original": {
@@ -1529,11 +1529,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1754102567,
+        "lastModified": 1754707163,
-        "narHash": "sha256-d6nZ+1e4VDqW6VAwfx9EAUDJdPxSwqwGiuli32FEgoE=",
+        "narHash": "sha256-wgVgOsyJUDn2ZRpzu2gELKALoJXlBSoZJSln+Tlg5Pw=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "08ff39bf869cadca3102b39824f4c7025186b7dc",
+        "rev": "ac39ab4c8ed7cefe48d5ae5750f864422df58f01",
         "type": "github"
       },
       "original": {
@@ -1607,11 +1607,11 @@
         "nixpkgs": "nixpkgs_9"
       },
       "locked": {
-        "lastModified": 1753556729,
+        "lastModified": 1754761025,
-        "narHash": "sha256-KnY7idadDb5Vx24q2KjAaiIJVyCVHJsV7om1qQLednA=",
+        "narHash": "sha256-Mo2BkJXIz6HKM8cX2S7bRdX6Q3E1UOcyVL4v10QEUzk=",
         "owner": "krebs",
         "repo": "stockholm",
-        "rev": "4ec2ff1c3f0854e113714542e9a2b48a80d70b62",
+        "rev": "fc32e4609140fffa1312a4ca1aeea550b7467448",
         "type": "github"
       },
       "original": {
@@ -1671,11 +1671,11 @@
         "tinted-zed": "tinted-zed"
       },
       "locked": {
-        "lastModified": 1753979771,
+        "lastModified": 1754599117,
-        "narHash": "sha256-MdMdQymbivEWWkC5HqeLYtP8FYu0SqiSpiRlyw9Fm3Y=",
+        "narHash": "sha256-AzAYdZlat002vCjCKWdFpGi2xUaiOU4DtIPnv1nomD8=",
         "owner": "danth",
         "repo": "stylix",
-        "rev": "5b81b0c4fbab3517b39d63f493760d33287150ad",
+        "rev": "312dec38b2231b21f36903d1bdce96daa11548ff",
         "type": "github"
       },
       "original": {

systems/makanek/monitoring/default.nix

@@ -3,11 +3,13 @@
   config,
   pkgs,
   ...
-}: let
+}:
+let
   lokiConfig = import ./loki.nix;
   blackboxConfig = import ./blackbox.nix;
   inherit (import ../../../lib) restic;
-in {
+in
+{
   services.grafana = {
     enable = true;
     settings = {
@@ -80,143 +82,150 @@ in {
     }
   ];
 
-  services.prometheus.rules = let
+  services.prometheus.rules =
-    diskFreeThreshold = 10;
+    let
-  in [
+      diskFreeThreshold = 10;
-    (builtins.toJSON {
+    in
-      groups = [
+    [
-        {
+      (builtins.toJSON {
-          name = "niveum";
+        groups = [
-          rules = [
+          {
-            {
+            name = "niveum";
-              alert = "HostSystemdServiceCrashed";
+            rules = [
-              expr = ''(node_systemd_unit_state{state="failed"} == 1) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}'';
+              {
-              annotations = {
+                alert = "HostSystemdServiceCrashed";
-                description = "{{$labels.name}} failed on {{$labels.instance}}";
+                expr = ''(node_systemd_unit_state{state="failed"} == 1) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}'';
-              };
+                annotations = {
-            }
+                  description = "{{$labels.name}} failed on {{$labels.instance}}";
-            {
+                };
-              alert = "RootPartitionFull";
+              }
-              for = "10m";
+              {
-              expr = ''(node_filesystem_free_bytes{mountpoint="/"} * 100) / node_filesystem_size_bytes{mountpoint="/"} < ${toString diskFreeThreshold}'';
+                alert = "RootPartitionFull";
-              annotations = {
+                for = "10m";
-                description = ''{{ $labels.instance }} running out of space: {{ $value | printf "%.2f" }}% < ${toString diskFreeThreshold}%'';
+                expr = ''(node_filesystem_free_bytes{mountpoint="/"} * 100) / node_filesystem_size_bytes{mountpoint="/"} < ${toString diskFreeThreshold}'';
-              };
+                annotations = {
-            }
+                  description = ''{{ $labels.instance }} running out of space: {{ $value | printf "%.2f" }}% < ${toString diskFreeThreshold}%'';
-            {
+                };
-              alert = "RootPartitionFullWeek";
+              }
-              for = "1h";
+              {
-              expr =
+                alert = "RootPartitionFullWeek";
-                ''node_filesystem_free_bytes{mountpoint="/"} ''
+                for = "1h";
-                + ''and predict_linear(node_filesystem_free_bytes{mountpoint="/"}[2d], 7*24*3600) <= 0'';
+                expr =
-              annotations = {
+                  ''node_filesystem_free_bytes{mountpoint="/"} ''
-                description = "{{$labels.instance}} running out of space in 7 days";
+                  + ''and predict_linear(node_filesystem_free_bytes{mountpoint="/"}[2d], 7*24*3600) <= 0'';
-              };
+                annotations = {
-            }
+                  description = "{{$labels.instance}} running out of space in 7 days";
-            {
+                };
-              alert = "HighLoad";
+              }
-              expr = ''node_load15 / on(job) count(node_cpu_seconds_total{mode="system"}) by (job) >= 1.0'';
+              {
-              for = "10m";
+                alert = "HighLoad";
-              annotations = {
+                expr = ''node_load15 / on(job) count(node_cpu_seconds_total{mode="system"}) by (job) >= 1.0'';
-                description = "{{$labels.instance}} running on high load: {{$value}}";
+                for = "10m";
-              };
+                annotations = {
-            }
+                  description = "{{$labels.instance}} running on high load: {{$value}}";
-            {
+                };
-              alert = "HostUnusualNetworkThroughputIn";
+              }
-              expr = ''(rate(node_network_receive_bytes_total[2m])) / 1024 / 1024 > 100'';
+              {
-              for = "5m";
+                alert = "HostUnusualNetworkThroughputIn";
-              annotations.description = "Host unusual network throughput in (instance {{ $labels.instance }})";
+                expr = ''(rate(node_network_receive_bytes_total[2m])) / 1024 / 1024 > 100'';
-            }
+                for = "5m";
-            {
+                annotations.description = "Host unusual network throughput in (instance {{ $labels.instance }})";
-              alert = "HostUnusualNetworkThroughputOut";
+              }
-              expr = ''(rate(node_network_transmit_bytes_total[2m])) / 1024 / 1024 > 100'';
+              {
-              for = "5m";
+                alert = "HostUnusualNetworkThroughputOut";
-              annotations.description = "Host unusual network throughput out (instance {{ $labels.instance }})";
+                expr = ''(rate(node_network_transmit_bytes_total[2m])) / 1024 / 1024 > 100'';
-            }
+                for = "5m";
-            {
+                annotations.description = "Host unusual network throughput out (instance {{ $labels.instance }})";
-              alert = "HostUnusualDiskReadRate";
+              }
-              expr = ''(rate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50'';
+              {
-              for = "5m";
+                alert = "HostUnusualDiskReadRate";
-              annotations.description = "Host unusual disk read rate (instance {{ $labels.instance }})";
+                expr = ''(rate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50'';
-            }
+                for = "5m";
-            {
+                annotations.description = "Host unusual disk read rate (instance {{ $labels.instance }})";
-              alert = "HostUnusualDiskWriteRate";
+              }
-              expr = ''(rate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50'';
+              {
-              for = "2m";
+                alert = "HostUnusualDiskWriteRate";
-              annotations.description = "Host unusual disk write rate (instance {{ $labels.instance }})";
+                expr = ''(rate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50'';
-            }
+                for = "2m";
-            {
+                annotations.description = "Host unusual disk write rate (instance {{ $labels.instance }})";
-              alert = "HostOutOfInodes";
+              }
-              expr = ''node_filesystem_files_free{fstype!="msdosfs"} / node_filesystem_files{fstype!="msdosfs"} * 100 < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0'';
+              {
-              for = "2m";
+                alert = "HostOutOfInodes";
-              annotations.description = "Host out of inodes (instance {{ $labels.instance }})";
+                expr = ''node_filesystem_files_free{fstype!="msdosfs"} / node_filesystem_files{fstype!="msdosfs"} * 100 < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0'';
-            }
+                for = "2m";
-            {
+                annotations.description = "Host out of inodes (instance {{ $labels.instance }})";
-              alert = "HostInodesWillFillIn24Hours";
+              }
-              expr = ''node_filesystem_files_free{fstype!="msdosfs"} / node_filesystem_files{fstype!="msdosfs"} * 100 < 10 and predict_linear(node_filesystem_files_free{fstype!="msdosfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly{fstype!="msdosfs"} == 0'';
+              {
-              for = "2m";
+                alert = "HostInodesWillFillIn24Hours";
-              annotations.description = "Host inodes will fill in 24 hours (instance {{ $labels.instance }})";
+                expr = ''node_filesystem_files_free{fstype!="msdosfs"} / node_filesystem_files{fstype!="msdosfs"} * 100 < 10 and predict_linear(node_filesystem_files_free{fstype!="msdosfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly{fstype!="msdosfs"} == 0'';
-            }
+                for = "2m";
-            {
+                annotations.description = "Host inodes will fill in 24 hours (instance {{ $labels.instance }})";
-              alert = "HighRAM";
+              }
-              expr = "node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes < node_memory_MemTotal_bytes * 0.1";
+              {
-              for = "1h";
+                alert = "HighRAM";
-              annotations.description = "{{$labels.instance}} using lots of RAM";
+                expr = "node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes < node_memory_MemTotal_bytes * 0.1";
-            }
+                for = "1h";
-            {
+                annotations.description = "{{$labels.instance}} using lots of RAM";
-              alert = "UptimeMonster";
+              }
-              expr = "time() - node_boot_time_seconds > 2592000";
+              {
-              annotations.description = "uptime monster {{$labels.instance}} up for more than 30 days";
+                alert = "UptimeMonster";
-            }
+                expr = "time() - node_boot_time_seconds > 2592000";
-            {
+                annotations.description = "uptime monster {{$labels.instance}} up for more than 30 days";
-              alert = "HostDown";
+              }
-              expr = ''up == 0'';
+              {
-              for = "5m";
+                alert = "HostDown";
-              annotations = {
+                expr = ''up == 0'';
-                description = "{{ $labels.instance }} seeming down since 5 minutes";
+                for = "5m";
-              };
+                annotations = {
-            }
+                  description = "{{ $labels.instance }} seeming down since 5 minutes";
-            {
+                };
-              alert = "Reboot";
+              }
-              expr = "time() - node_boot_time_seconds < 300";
+              {
-              annotations.description = "{{$labels.instance}} rebooted";
+                alert = "Reboot";
-            }
+                expr = "time() - node_boot_time_seconds < 300";
-            {
+                annotations.description = "{{$labels.instance}} rebooted";
-              alert = "ProbeFailed";
+              }
-              expr = "probe_success == 0";
+              {
-              for = "5m";
+                alert = "Mastodon";
-              annotations.description = "HTTP probe failed for {{$labels.instance}}";
+                expr = ''probe_success{instance="https://social.krebsco.de"}'';
-            }
+                for = "5m";
-            {
+                annotations.description = "Mastodon instance {{$labels.instance}} is down";
-              alert = "SlowProbe";
+              }
-              expr = "avg_over_time(probe_http_duration_seconds[1m]) > 1";
+              {
-              for = "5m";
+                alert = "ProbeFailed";
-              annotations.description = "HTTP probe slow for {{$labels.instance}}";
+                expr = "probe_success == 0";
-            }
+                for = "5m";
-            {
+                annotations.description = "HTTP probe failed for {{$labels.instance}}";
-              alert = "HttpStatusCode";
+              }
-              expr = "probe_http_status_code != 0 AND (probe_http_status_code <= 199 OR probe_http_status_code >= 400)";
+              {
-              for = "5m";
+                alert = "SlowProbe";
-              annotations.description = "status code {{$value}} for {{$labels.instance}}";
+                expr = "avg_over_time(probe_http_duration_seconds[1m]) > 1";
-            }
+                for = "5m";
-            {
+                annotations.description = "HTTP probe slow for {{$labels.instance}}";
-              alert = "SslExpirySoon";
+              }
-              expr = "probe_ssl_earliest_cert_expiry - time() < 86400 * 30";
+              {
-              for = "5m";
+                alert = "HttpStatusCode";
-              annotations.description = "SSL certificate for {{$labels.instance}} expires in 30 days";
+                expr = "probe_http_status_code != 0 AND (probe_http_status_code <= 199 OR probe_http_status_code >= 400)";
-            }
+                for = "5m";
-            {
+                annotations.description = "status code {{$value}} for {{$labels.instance}}";
-              alert = "SslExpiry";
+              }
-              expr = "probe_ssl_earliest_cert_expiry - time()  <= 0";
+              {
-              for = "5m";
+                alert = "SslExpirySoon";
-              annotations.description = "SSL certificate for {{$labels.instance}} has expired";
+                expr = "probe_ssl_earliest_cert_expiry - time() < 86400 * 30";
-            }
+                for = "5m";
-          ];
+                annotations.description = "SSL certificate for {{$labels.instance}} expires in 30 days";
-        }
+              }
-      ];
+              {
-    })
+                alert = "SslExpiry";
-  ];
+                expr = "probe_ssl_earliest_cert_expiry - time()  <= 0";
-
+                for = "5m";
+                annotations.description = "SSL certificate for {{$labels.instance}} has expired";
+              }
+            ];
+          }
+        ];
+      })
+    ];
 
   # ref https://github.com/Mic92/dotfiles/blob/f44bac5dd6970ed3fbb4feb906917331ec3c2be5/machines/eva/modules/prometheus/default.nix
   systemd.services.matrix-hook = {
@@ -246,6 +255,33 @@ in {
     };
   };
 
+  systemd.services.matrix-hook-lassulus = {
+    description = "Matrix Hook";
+    after = [ "network.target" ];
+    wantedBy = [ "multi-user.target" ];
+    environment = {
+      HTTP_ADDRESS = "[::1]";
+      HTTP_PORT = "9089";
+      MX_HOMESERVER = "https://matrix.4d2.org";
+      MX_ID = "@lakai:4d2.org";
+      MX_ROOMID = "!MJAGqBAOKZGMywzwkI:lassul.us";
+      MX_MSG_TEMPLATE = "${pkgs.matrix-hook}/message.html.tmpl";
+    };
+    serviceConfig = {
+      EnvironmentFile = [
+        # format: MX_TOKEN=<token>
+        config.age.secrets.matrix-token-lakai-env.path
+      ];
+      Type = "simple";
+      ExecStart = "${pkgs.matrix-hook}/bin/matrix-hook";
+      Restart = "always";
+      RestartSec = "10";
+      DynamicUser = true;
+      User = "matrix-hook";
+      Group = "matrix-hook";
+    };
+  };
+
   age.secrets = {
     matrix-token-lakai-env.file = ../../../secrets/matrix-token-lakai-env.age;
   };
@@ -260,8 +296,23 @@ in {
         group_wait = "30s";
         repeat_interval = "24h";
         receiver = "matrix";
+        routes = [
+          {
+            receiver = "lassulus";
+            matchers = [ "alertname = \"Mastodon\"" ];
+          }
+        ];
       };
       receivers = [
+        {
+          name = "lassulus";
+          webhook_configs = [
+            {
+              url = "http://localhost:9089/alert";
+              max_alerts = 5;
+            }
+          ];
+        }
         {
           name = "matrix";
           webhook_configs = [
@@ -306,13 +357,21 @@ in {
     {
       scheme = "http";
       path_prefix = "/";
-      static_configs = [{targets = ["localhost:${toString config.services.prometheus.alertmanager.port}"];}];
+      static_configs = [
+        { targets = [ "localhost:${toString config.services.prometheus.alertmanager.port}" ]; }
+      ];
     }
   ];
 
   # otherwise bearer_token_file will fail
   services.prometheus.checkConfig = "syntax-only";
 
+  services.prometheus.extraFlags = [
+    "--storage.tsdb.retention.time=7d"
+    "--storage.tsdb.retention.size=2GB"
+    "--storage.tsdb.wal-compression"
+  ];
+
   services.prometheus.scrapeConfigs = [
     {
       job_name = "makanek";
@@ -328,14 +387,14 @@ in {
       scrape_interval = "5m";
       job_name = "blackbox";
       metrics_path = "/probe";
-      params.module = ["http_2xx"];
+      params.module = [ "http_2xx" ];
       relabel_configs = [
         {
-          source_labels = ["__address__"];
+          source_labels = [ "__address__" ];
           target_label = "__param_target";
         }
         {
-          source_labels = ["__param_target"];
+          source_labels = [ "__param_target" ];
           target_label = "instance";
         }
         {
@@ -393,7 +452,7 @@ in {
       scrape_interval = "60s";
       metrics_path = "/api/prometheus";
       scheme = "http";
-      static_configs = [{targets = ["zaatar.r:8123"];}];
+      static_configs = [ { targets = [ "zaatar.r:8123" ]; } ];
       bearer_token_file = config.age.secrets.home-assistant-token.path;
     }
     {
@@ -410,7 +469,7 @@ in {
 
   services.prometheus.exporters.blackbox = {
     enable = true;
-    configFile = (pkgs.formats.yaml {}).generate "blackbox.yaml" blackboxConfig;
+    configFile = (pkgs.formats.yaml { }).generate "blackbox.yaml" blackboxConfig;
   };
 
   networking.firewall.allowedTCPPorts = [
@@ -419,6 +478,6 @@ in {
 
   services.loki = {
     enable = true;
-    configFile = (pkgs.formats.yaml {}).generate "loki.yaml" lokiConfig;
+    configFile = (pkgs.formats.yaml { }).generate "loki.yaml" lokiConfig;
   };
 }