chore: move configs to system directories

2026-03-16 18:21:07 +01:00 · 2021-12-29 18:49:51 +01:00
parent 383fb35837
commit 58d39aa41a
43 changed files with 234 additions and 689 deletions
--- a/systems/kabsa/configuration.nix
+++ b/systems/kabsa/configuration.nix
@@ -5,11 +5,10 @@ let
 in
 {
  imports = [
-    <niveum/configs/default.nix>
-    <niveum/configs/battery.nix>
-    <niveum/configs/networkmanager.nix>
    ./hardware-configuration.nix
-    # <stockholm/krebs/2configs/hw/x220.nix>
+    <niveum/configs/battery.nix>
+    <niveum/configs/default.nix>
+    <niveum/configs/networkmanager.nix>
  ];

  niveum = {
@@ -18,29 +17,18 @@ in
    promptColours.success = "cyan";
  };

-  nix.buildCores = 1;
-  nix.maxJobs = 2;
+  nix = {
+    buildCores = 1;
+    maxJobs = 2;
+  };

  environment.systemPackages = [ nixpkgs-unstable.minecraft ];

-  boot.loader.systemd-boot = {
-    enable = true;
-    configurationLimit = 5;
-    consoleMode = "max";
+  networking = {
+    hostName = "kabsa";
+    wireless.interfaces = [ "wlp3s0" ];
+    retiolum = retiolumAddresses.kabsa;
  };

-  fileSystems."/mnt/sd-card" = {
-    device = "/dev/disk/by-id/mmc-SD32G_0xda0aa352-part1";
-    fsType = "vfat";
-  };
-
-  boot.loader.efi.canTouchEfiVariables = true;
-
-  networking.hostName = "kabsa";
-
-  networking.wireless.interfaces = [ "wlp3s0" ];
-
-  networking.retiolum = retiolumAddresses.kabsa;
-
  system.stateVersion = "19.03";
 }
--- a/systems/kabsa/hardware-configuration.nix
+++ b/systems/kabsa/hardware-configuration.nix
@@ -1,21 +1,37 @@
 { config, lib, pkgs, ... }: {
  imports = [ <nixpkgs/nixos/modules/installer/scan/not-detected.nix> ];

-  boot.initrd.availableKernelModules = [ "ehci_pci" "ahci" "xhci_pci" "usb_storage" "sd_mod" "sdhci_pci" ];
-  boot.initrd.kernelModules = [ ];
-  boot.kernelModules = [ "kvm-intel" ];
-  boot.extraModulePackages = [ ];
-
-  boot.initrd.luks.devices."luksmap".device = "/dev/disk/by-uuid/03b6abd0-e9ce-49c8-9659-a1d94f645d0f";
-
-  fileSystems."/" = {
-    device = "/dev/disk/by-uuid/66ced7b9-cfa7-40dd-a488-18ef91a337e0";
-    fsType = "ext4";
+  boot = {
+    initrd = {
+      availableKernelModules = [ "ehci_pci" "ahci" "xhci_pci" "usb_storage" "sd_mod" "sdhci_pci" ];
+      luks.devices."luksmap".device = "/dev/disk/by-uuid/03b6abd0-e9ce-49c8-9659-a1d94f645d0f";
+      kernelModules = [ ];
+    };
+    kernelModules = [ "kvm-intel" ];
+    extraModulePackages = [ ];
+    loader = {
+      efi.canTouchEfiVariables = true;
+      systemd-boot = {
+        enable = true;
+        configurationLimit = 5;
+        consoleMode = "max";
+      };
+    };
  };

-  fileSystems."/boot" = {
-    device = "/dev/disk/by-uuid/E1B1-1B9C";
-    fsType = "vfat";
+  fileSystems = {
+    "/" = {
+      device = "/dev/disk/by-uuid/66ced7b9-cfa7-40dd-a488-18ef91a337e0";
+      fsType = "ext4";
+    };
+    "/boot" = {
+      device = "/dev/disk/by-uuid/E1B1-1B9C";
+      fsType = "vfat";
+    };
+    "/mnt/sd-card" = {
+      device = "/dev/disk/by-id/mmc-SD32G_0xda0aa352-part1";
+      fsType = "vfat";
+    };
  };

  zramSwap.enable = false;
--- a/systems/makanek/configuration.nix
+++ b/systems/makanek/configuration.nix
@@ -4,70 +4,47 @@ let
 in
 {
  imports = [
+    ./gitea.nix
    ./hardware-configuration.nix
-    <niveum/configs/hedgedoc.nix>
+    ./hedgedoc.nix
+    ./matterbridge.nix
+    ./menstruation.nix
+    ./monitoring
+    ./moodle-dl-borsfaye.nix
+    ./names.nix
+    ./nextcloud.nix
+    ./radio
+    ./retiolum-map.nix
+    ./tarot.nix
+    ./urlwatch.nix
+    ./weechat.nix
+    <niveum/configs/monitoring.nix>
+    <niveum/configs/nix.nix>
+    <niveum/configs/save-space.nix>
    <niveum/configs/spacetime.nix>
    <niveum/configs/sshd.nix>
-    <niveum/configs/nextcloud.nix>
-    <niveum/configs/moodle-dl/borsfaye.nix>
-    <niveum/configs/save-space.nix>
-    <niveum/configs/monitoring/pull.nix>
-    <niveum/configs/monitoring/push.nix>
-    <niveum/configs/version.nix>
-    <niveum/configs/radio>
-    <niveum/configs/gitea.nix>
-    <niveum/configs/retiolum-map.nix>
-    <niveum/configs/names.nix>
-    <niveum/configs/menstruation.nix>
    <niveum/configs/telegram-bots>
-    <niveum/configs/nix.nix>
-    <niveum/configs/weechat.nix>
-    <niveum/configs/urlwatch.nix>
-    <niveum/configs/matterbridge.nix>
-    <niveum/configs/tarot.nix>
+    <niveum/configs/version.nix>
    <niveum/modules/retiolum.nix>
  ];

-  boot.loader.grub.enable = true;
-  boot.loader.grub.version = 2;
-
-  nixpkgs.config = {
-    allowUnfree = true;
-    packageOverrides = pkgs: {
-      writeDashBin = pkgs.writers.writeDashBin;
-      writeDash = pkgs.writers.writeDash;
-    };
+  networking = {
+    firewall.allowedTCPPorts = [ 80 443 ];
+    hostName = "makanek";
+    interfaces.ens3.useDHCP = true;
+    retiolum = retiolumAddresses.makanek;
+    useDHCP = false;
  };

-  networking.useDHCP = false;
-  networking.interfaces.ens3.useDHCP = true;
-
-  networking.hostName = "makanek";
-
  system.stateVersion = "20.03";

-  boot.loader.grub.devices = [ "/dev/sda" ];
-
-  services.openssh.enable = true;
-
-  networking.retiolum = retiolumAddresses.makanek;
-
-  environment.etc."tinc/retiolum/rsa_key.priv" = {
-    text = builtins.readFile <system-secrets/retiolum.key>;
-    mode = "400";
-  };
-
-  networking.firewall.allowedTCPPorts = [ 80 443 ];
-
  services.nginx = {
    enable = true;
    recommendedGzipSettings = true;
    recommendedOptimisation = true;
    recommendedProxySettings = true;
    recommendedTlsSettings = true;
-
-   # Only allow PFS-enabled ciphers with AES256
-   sslCiphers = "AES256+EECDH:AES256+EDH:!aNULL";
+    sslCiphers = "AES256+EECDH:AES256+EDH:!aNULL";
  };

  security.acme = {
@@ -75,5 +52,5 @@ in
    email = kieran.email;
  };

-  environment.systemPackages = [ pkgs.vim pkgs.git pkgs.tmux pkgs.python3Packages.python ];
+  environment.systemPackages = [ pkgs.vim pkgs.git pkgs.tmux pkgs.python3 ];
 }
--- a/systems/makanek/gitea.nix
+++ b/systems/makanek/gitea.nix
@@ -0,0 +1,13 @@
+{
+  services.gitea = {
+    enable = true;
+    disableRegistration = true;
+    rootUrl = "https://code.kmein.de";
+    appName = "code.kmein.de";
+  };
+  services.nginx.virtualHosts."code.kmein.de" = {
+    forceSSL = true;
+    enableACME = true;
+    locations."/".extraConfig = "proxy_pass http://localhost:3000;";
+  };
+}
--- a/systems/makanek/hardware-configuration.nix
+++ b/systems/makanek/hardware-configuration.nix
@@ -1,19 +1,25 @@
 { config, lib, pkgs, ... }:
-
 {
-  imports =
-    [ <nixpkgs/nixos/modules/profiles/qemu-guest.nix>
-    ];
+  imports = [ <nixpkgs/nixos/modules/profiles/qemu-guest.nix> ];

-  boot.initrd.availableKernelModules = [ "ata_piix" "virtio_pci" "xhci_pci" "sd_mod" "sr_mod" ];
-  boot.initrd.kernelModules = [ ];
-  boot.kernelModules = [ ];
-  boot.extraModulePackages = [ ];
-
-  fileSystems."/" =
-    { device = "/dev/disk/by-uuid/9eaeaaa7-b453-4634-8a69-d416f702d3aa";
-      fsType = "ext4";
+  boot = {
+    initrd = {
+      availableKernelModules = [ "ata_piix" "virtio_pci" "xhci_pci" "sd_mod" "sr_mod" ];
+      kernelModules = [ ];
    };
+    kernelModules = [ ];
+    extraModulePackages = [ ];
+    loader.grub = {
+      enable = true;
+      version = 2;
+      devices = [ "/dev/sda" ];
+    };
+  };
+
+  fileSystems."/" = {
+    device = "/dev/disk/by-uuid/9eaeaaa7-b453-4634-8a69-d416f702d3aa";
+    fsType = "ext4";
+  };

  swapDevices = [ ];
  zramSwap.enable = true;
--- a/systems/makanek/hedgedoc.nix
+++ b/systems/makanek/hedgedoc.nix
@@ -0,0 +1,81 @@
+{ config, pkgs, ... }:
+let
+  backupLocation = "/var/lib/codimd-backup";
+  stateLocation = "/var/lib/codimd/state.sqlite";
+  nixpkgs-unstable = import <nixpkgs-unstable> {};
+  domain = "pad.kmein.de";
+in
+{
+  imports = [ <stockholm/krebs/3modules/permown.nix> ];
+
+  services.nginx.virtualHosts.${domain} = {
+    enableACME = true;
+    forceSSL = true;
+    locations."/" = {
+      proxyPass = "https://localhost:3091";
+      proxyWebsockets = true;
+    };
+  };
+
+  security.acme.certs.${domain}.group = "hedgecert";
+  users.groups.hedgecert.members = [ "codimd" "nginx" ];
+
+  security.dhparams = {
+    enable = true;
+    params.hedgedoc = {};
+  };
+
+  services.hedgedoc = {
+    enable = true;
+    configuration = {
+      allowOrigin = [ domain ];
+      allowAnonymous = true;
+      allowGravatar = false;
+      allowFreeURL = true;
+      db = {
+        dialect = "sqlite";
+        storage = stateLocation;
+      };
+      port = 3091;
+      domain = domain;
+      useSSL = true;
+      protocolUseSSL = true;
+      sslCAPath = [ "/etc/ssl/certs/ca-certificates.crt" ];
+      sslCertPath = "/var/lib/acme/${domain}/cert.pem";
+      sslKeyPath = "/var/lib/acme/${domain}/key.pem";
+      dhParamPath = config.security.dhparams.params.hedgedoc.path;
+    };
+  };
+
+  krebs.permown.${backupLocation} = { owner = "codimd"; group = "codimd"; umask = "0002"; };
+
+  systemd.services.hedgedoc-backup = {
+    description = "Hedgedoc backup service";
+    script = ''
+      ${nixpkgs-unstable.sqlite}/bin/sqlite3 -json ${stateLocation} "select shortid, alias, ownerId, content from Notes" \
+      | ${pkgs.writers.writePython3 "hedgedoc-json-to-fs.py" {} ''
+        import json
+        import pathlib
+        import sys
+
+        for note in json.load(sys.stdin):
+            user_directory = pathlib.Path()
+            if note["ownerId"]:
+                user_directory = pathlib.Path(note["ownerId"])
+                user_directory.mkdir(exist_ok=True)
+            file_path = user_directory / (
+                (note["alias"] if note["alias"] else note["shortid"]) + ".md"
+            )
+            file_path.write_text(note["content"])
+            print(f"✔ {file_path}", file=sys.stderr)
+      ''}
+    '';
+    startAt = "hourly";
+    serviceConfig = {
+      Type = "oneshot";
+      User = "codimd";
+      Group = "codimd";
+      WorkingDirectory = backupLocation;
+    };
+  };
+}
--- a/systems/makanek/matterbridge.nix
+++ b/systems/makanek/matterbridge.nix
@@ -0,0 +1,56 @@
+{ pkgs, lib, ... }:
+{
+  nixpkgs.overlays = [
+    (self: super: {
+      matterbridge = (import (super.fetchFromGitHub {
+        owner = "NixOS";
+        repo = "nixpkgs";
+        rev = "e45d91ee65db293a172ec506759d1248e40c35f5";
+        sha256 = "03cjs5xcx09lw0djyrx2kfakw7gkg4iqmy9w25azai62im39l30k";
+      }) {}).matterbridge;
+    })
+  ];
+
+  services.matterbridge = {
+    enable = true;
+    configPath =
+    let bridgeBotToken = lib.strings.fileContents <system-secrets/telegram/kmein.token>;
+    in toString ((pkgs.formats.toml {}).generate "config.toml" {
+      general = {
+        RemoteNickFormat = "[{NICK}] ";
+        Charset = "utf-8";
+      };
+      telegram.kmein.Token = bridgeBotToken;
+      irc =
+      let Nick = "ponte";
+      in {
+        hackint = { Server = "irc.hackint.org:6697"; UseTLS = true; inherit Nick; };
+      };
+      mumble.lassulus = {
+        Server = "lassul.us:64738";
+        Nick = "krebs_bridge";
+        SkipTLSVerify = true;
+      };
+      gateway = [
+        {
+          name = "krebs-bridge";
+          enable = true;
+          inout = [
+            {
+              account = "irc.hackint";
+              channel = "#krebs";
+            }
+            {
+              account = "telegram.kmein";
+              channel = "-330372458";
+            }
+            {
+              account = "mumble.lassulus";
+              channel = 6; # "nixos"
+            }
+          ];
+        }
+      ];
+    });
+  };
+}
--- a/systems/makanek/menstruation.nix
+++ b/systems/makanek/menstruation.nix
@@ -0,0 +1,41 @@
+{ pkgs, lib, ... }:
+let
+  backend = pkgs.callPackage <niveum/submodules/menstruation-backend> {};
+  telegram = pkgs.callPackage <niveum/submodules/menstruation-telegram> {};
+  backendPort = 8000;
+in
+{
+  containers.menstruation.autoStart = true;
+  containers.menstruation.config = { config, pkgs, ... }: {
+    services.redis.enable = true;
+
+    systemd.services.menstruation-telegram = {
+      wants = [
+        "network-online.target"
+        "menstruation-backend.service"
+        "redis.service"
+      ];
+      wantedBy = [ "multi-user.target" ];
+      environment = {
+        MENSTRUATION_TOKEN = lib.strings.fileContents <system-secrets/telegram/menstruation.token>;
+        MENSTRUATION_ENDPOINT = "http://localhost:${toString backendPort}";
+        MENSTRUATION_MODERATORS = "18980945";
+      };
+      serviceConfig = {
+        Restart = "always";
+        DynamicUser = true;
+        ExecStart = "${telegram}/bin/menstruation-telegram";
+      };
+    };
+
+    systemd.services.menstruation-backend = {
+      wants = [ "network-online.target" ];
+      environment.ROCKET_PORT = toString backendPort;
+      serviceConfig = {
+        Restart = "always";
+        DynamicUser = true;
+        ExecStart = "${backend}/bin/menstruation_server";
+      };
+    };
+  };
+}
--- a/systems/makanek/monitoring/blackbox.nix
+++ b/systems/makanek/monitoring/blackbox.nix
@@ -0,0 +1,17 @@
+# https://github.com/Fluepke/nix-files/blob/2be70b76a198afaa7763132fed645a3c19d5af6e/configuration/common/blackbox-exporter.yml
+# https://github.com/xHain-hackspace/xhain-nixfiles/blob/0d6e3b87a07317c2d54cccabf4f90da589319e2c/common/prometheus/blackbox-exporter.yml
+{
+  modules.http_2xx = {
+    http = {
+      fail_if_not_ssl = true;
+      ip_protocol_fallback = false;
+      method = "GET";
+      no_follow_redirects = false;
+      preferred_ip_protocol = "ip4";
+      valid_http_versions = [ "HTTP/1.1" "HTTP/2.0" ];
+      tls_config.insecure_skip_verify = true;
+    };
+    prober = "http";
+    timeout = "15s";
+  };
+}
--- a/systems/makanek/monitoring/default.nix
+++ b/systems/makanek/monitoring/default.nix
@@ -0,0 +1,229 @@
+{ lib, config, pkgs, ... }:
+let
+  lokiConfig = import ./loki.nix;
+  blackboxConfig = import ./blackbox.nix;
+in
+{
+  services.grafana = {
+    enable = true;
+    domain = "grafana.kmein.r";
+    port = 9444;
+    addr = "127.0.0.1";
+  };
+
+  services.nginx.virtualHosts.${config.services.grafana.domain} = {
+    locations."/" = {
+      proxyPass = "http://127.0.0.1:${toString config.services.grafana.port}";
+      proxyWebsockets = true;
+    };
+  };
+
+  services.prometheus.rules = let diskFreeThreshold = 10; in [(builtins.toJSON {
+    groups = [{
+      name = "niveum";
+      rules = [
+        {
+          alert = "ServiceDown";
+          expr = ''node_systemd_unit_state{state="failed"} == 1'';
+          annotations = {
+            summary = "{{$labels.job}}: Service {{$labels.name}} failed to start.";
+          };
+        }
+        {
+          alert = "RootPartitionFull";
+          for = "10m";
+          expr = ''(node_filesystem_free_bytes{mountpoint="/"} * 100) / node_filesystem_size_bytes{mountpoint="/"} < ${toString diskFreeThreshold}'';
+          annotations = {
+            summary = "{{ $labels.job }}: Filesystem is running out of space soon.";
+            description = ''The root disk of {{ $labels.job }} has {{ $value | printf "%.2f" }}% free disk space (threshold at ${toString diskFreeThreshold}%).'';
+          };
+        }
+        {
+          alert = "RootPartitionFullWeek";
+          for = "1h";
+          expr = ''node_filesystem_free_bytes{mountpoint="/"} ''
+            + ''and predict_linear(node_filesystem_free_bytes{mountpoint="/"}[2d], 7*24*3600) <= 0'';
+          annotations = {
+            summary = "{{$labels.job}}: Filesystem is running out of space in 7 days.";
+          };
+        }
+        {
+          alert = "HighLoad";
+          expr = ''node_load15 / on(job) count(node_cpu_seconds_total{mode="system"}) by (job) >= 1.0'';
+          for = "10m";
+          annotations = {
+            summary = "{{$labels.job}}: Running on high load: {{$value}}";
+          };
+        }
+        {
+          alert = "HighRAM";
+          expr = "node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes < node_memory_MemTotal_bytes * 0.1";
+          for = "1h";
+          annotations.summary = "{{$labels.job}}: Using lots of RAM.";
+        }
+        {
+          alert = "UptimeMonster";
+          expr = "time() - node_boot_time_seconds > 2592000";
+          annotations.summary = "{{$labels.job}}: up for more than 30 days.";
+        }
+        {
+          alert = "HostDown";
+          expr = ''up == 0'';
+          for = "5m";
+          annotations = {
+            summary = "Host {{ $labels.job }} down for 5 minutes.";
+          };
+        }
+        {
+          alert = "Reboot";
+          expr = "time() - node_boot_time_seconds < 300";
+          annotations.summary = "{{$labels.job}}: Reboot";
+        }
+        {
+          alert = "ProbeFailed";
+          expr = "probe_success == 0";
+          for = "5m";
+          annotations.summary = "{{$labels.instance}}: probe failed";
+        }
+        {
+          alert = "SlowProbe";
+          expr = "avg_over_time(probe_http_duration_seconds[1m]) > 1";
+          for = "5m";
+          annotations.summary = "{{$labels.instance}}: HTTP probe slow";
+        }
+        {
+          alert = "HttpStatusCode";
+          expr = "probe_http_status_code != 0 AND (probe_http_status_code <= 199 OR probe_http_status_code >= 400)";
+          for = "5m";
+          annotations.summary = "{{$labels.instance}}: status code {{$value}}";
+        }
+        {
+          alert = "SslExpirySoon";
+          expr = "probe_ssl_earliest_cert_expiry - time() < 86400 * 30";
+          for = "5m";
+          annotations.summary = "{{$labels.instance}}: SSL certificate expires in 30 days";
+        }
+        {
+          alert = "SslExpiry";
+          expr = "probe_ssl_earliest_cert_expiry - time()  <= 0";
+          for = "5m";
+          annotations.summary = "{{$labels.instance}}: SSL certificate has expired";
+        }
+      ];
+    }];
+  })];
+
+  systemd.services.alertmanager-bot-telegram =
+  let
+    alertmanager-bot-telegram = pkgs.buildGoModule rec {
+      pname = "alertmanager-bot";
+      version = "2020-07-13";
+      src = pkgs.fetchFromGitHub {
+        owner = "metalmatze";
+        repo = "alertmanager-bot";
+        rev = "5efc0bbbf8023d4324e9da98562f064a714a7206";
+        sha256 = "09cciml1j8x76jpm2v5v6h2q6j1fkhsz1kswslmx8wl4wk40xgp4";
+      };
+      vendorSha256 = "1v0fgin8dn81b559zz4lqmrl7hikr46g4gb18sci4riql5qs1isj";
+      postInstall = ''
+        install -D ./default.tmpl $out/templates/default.tmpl
+      '';
+    };
+  in {
+    wantedBy = [ "multi-user.target" ];
+    after = [ "ip-up.target" ];
+    environment.TELEGRAM_ADMIN = "18980945";
+    environment.TELEGRAM_TOKEN = lib.strings.fileContents <system-secrets/telegram/prometheus.token>;
+    serviceConfig = {
+      DynamicUser = true;
+      StateDirectory = "alertbot";
+      ExecStart = ''${alertmanager-bot-telegram}/bin/alertmanager-bot \
+        --alertmanager.url=http://localhost:9093 --log.level=info \
+        --store=bolt --bolt.path=/var/lib/alertbot/bot.db \
+        --listen.addr="0.0.0.0:16320" \
+        --template.paths=${pkgs.writeText "template.tmpl" ''
+          {{ define "telegram.default" }}
+          {{range .Alerts -}}
+          {{ if eq .Status "firing" }}
+          ⚠ <b>{{ index .Annotations "summary"}}</b>
+          {{ index .Annotations "description" }}
+
+          See on Grafana: http://${config.services.grafana.domain}/d/alpUteInz/niveum
+          {{ else -}}
+          RESOLVED 😌 <del>{{ index .Annotations "summary"}}</del>
+          {{- end }}
+          {{end -}}
+          {{end}}
+        ''}'';
+    };
+  };
+
+  services.prometheus.alertmanager = {
+    enable = true;
+    listenAddress = "localhost";
+    configuration = {
+      route = {
+        group_wait = "30s";
+        repeat_interval = "4h";
+        receiver = "me";
+      };
+      receivers = [{
+        name = "me";
+        webhook_configs = [{
+          url = "http://localhost:16320";
+          send_resolved = true;
+        }];
+      }];
+    };
+  };
+
+  services.prometheus.alertmanagers = [{
+    scheme = "http";
+    path_prefix = "/";
+    static_configs = [ { targets = [ "localhost:9093" ]; } ];
+  }];
+
+  services.prometheus.scrapeConfigs = [
+    {
+      job_name = "makanek";
+      static_configs = [ { targets = [
+        "127.0.0.1:${toString config.services.prometheus.exporters.node.port}"
+      ]; } ];
+    }
+    {
+      scrape_interval = "5m";
+      job_name = "blackbox";
+      metrics_path = "/probe";
+      params.module = [ "http_2xx" ];
+      relabel_configs = [
+        { source_labels = ["__address__"]; target_label = "__param_target"; }
+        { source_labels = ["__param_target"]; target_label = "instance"; }
+        { replacement = "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}"; target_label = "__address__"; }
+      ];
+      static_configs = [{
+        targets = [
+          "alew.hu-berlin.de"
+        ];
+      }];
+    }
+    {
+      job_name = "zaatar";
+      static_configs = [ { targets = [ "zaatar.r:${toString config.services.prometheus.exporters.node.port}" ]; } ];
+    }
+  ];
+
+
+  services.prometheus.exporters.blackbox = {
+    enable = true;
+    configFile = (pkgs.formats.yaml {}).generate "blackbox.yaml" blackboxConfig;
+  };
+
+  networking.firewall.allowedTCPPorts = [
+    lokiConfig.server.http_listen_port
+  ];
+
+  services.loki = {
+    enable = true;
+    configFile = (pkgs.formats.yaml {}).generate "loki.yaml" lokiConfig;
+  };
+}
--- a/systems/makanek/monitoring/loki.nix
+++ b/systems/makanek/monitoring/loki.nix
@@ -0,0 +1,60 @@
+{
+  auth_enabled = false;
+  server = {
+    http_listen_port = 3100;
+    grpc_listen_port = 9096;
+  };
+  ingester = {
+    wal = {
+      enabled = true;
+      dir = "/tmp/wal";
+    };
+    lifecycler = {
+      address = "127.0.0.1";
+      ring = {
+        kvstore.store = "inmemory";
+        replication_factor = 1;
+      };
+      final_sleep = "0s";
+    };
+    chunk_idle_period = "1h";       # Any chunk not receiving new logs in this time will be flushed
+    max_chunk_age = "1h";           # All chunks will be flushed when they hit this age, default is 1h
+    chunk_target_size = 1048576;    # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
+    chunk_retain_period = "30s";    # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
+    max_transfer_retries = 0;       # Chunk transfers disabled
+  };
+  schema_config.configs = [
+    {
+      from = "2020-10-24";
+      store = "boltdb-shipper";
+      object_store = "filesystem";
+      schema = "v11";
+      index = {
+        prefix = "index_";
+        period = "24h";
+      };
+    }
+  ];
+  storage_config = {
+    boltdb_shipper = {
+      active_index_directory = "/tmp/loki/boltdb-shipper-active";
+      cache_location = "/tmp/loki/boltdb-shipper-cache";
+      cache_ttl = "24h";         # Can be increased for faster performance over longer query periods, uses more disk space
+      shared_store = "filesystem";
+    };
+    filesystem.directory = "/tmp/loki/chunks";
+  };
+  compactor = {
+    working_directory = "/tmp/loki/boltdb-shipper-compactor";
+    shared_store = "filesystem";
+  };
+  limits_config = {
+    reject_old_samples = true;
+    reject_old_samples_max_age = "168h";
+  };
+  chunk_store_config.max_look_back_period = "0s";
+  table_manager = {
+    retention_deletes_enabled = false;
+    retention_period = "0s";
+  };
+}
--- a/systems/makanek/moodle-dl-borsfaye.nix
+++ b/systems/makanek/moodle-dl-borsfaye.nix
@@ -0,0 +1,31 @@
+{ config, pkgs, lib, ... }:
+let
+  moodle-dl-package = pkgs.moodle-dl.overrideAttrs (old: old // {
+    patches = [ <niveum/packages/moodle-dl/telegram-format.patch> ];
+  });
+in
+{
+  containers.moodle-dl-borsfaye = {
+    autoStart = true;
+    config = { lib, pkgs, ...}: {
+      imports = [ <niveum/modules/moodle-dl.nix> ];
+
+      services.moodle-dl = {
+        enable = true;
+        startAt = "hourly";
+        package = moodle-dl-package;
+        notifyOnly = true;
+        settings = {
+          telegram = {
+            token = lib.strings.fileContents <system-secrets/telegram/moodle-dl.token>;
+            chat_id = "311425510";
+            send_error_msg = false;
+          };
+          token = lib.strings.fileContents <system-secrets/moodle-dl/faye.token>;
+          moodle_domain = "moodle.hu-berlin.de";
+          moodle_path = "/";
+        };
+      };
+    };
+  };
+}
--- a/systems/makanek/names.nix
+++ b/systems/makanek/names.nix
@@ -0,0 +1,38 @@
+{ pkgs, lib, ... }:
+let
+  port = 5703;
+  geogen = (pkgs.fetchFromGitHub {
+    owner = "kmein";
+    repo = "scripts";
+    rev = "8945430f27a8c6fd632dd35382cb094abe3543ff";
+    sha256 = "1djyxkynypxsrmdf6idgjszqpcgqyq607rrsvl58p2bpymmwibzb";
+  }) + "/onomastics";
+  inherit (pkgs.callPackage geogen {}) dependencyEnv;
+in
+{
+  systemd.services.names = {
+    wants = [ "network-online.target" ];
+    wantedBy = [ "multi-user.target" ];
+    description = "Better clone of geogen.stoepel.net";
+    serviceConfig = {
+      DynamicUser = true;
+    };
+    script = ''
+      cd $(mktemp -d)
+      ln -s "${geogen}/wsgi.py" wsgi.py
+      ${dependencyEnv}/bin/gunicorn wsgi:app -b :${toString port}
+    '';
+  };
+
+  services.nginx = {
+    enable = true;
+    recommendedGzipSettings = true;
+    recommendedOptimisation = true;
+    recommendedProxySettings = true;
+    recommendedTlsSettings = true;
+  };
+
+  services.nginx.virtualHosts."names.kmein.r" = {
+    locations."/".proxyPass = "http://127.0.0.1:${toString port}";
+  };
+}
--- a/systems/makanek/nextcloud.nix
+++ b/systems/makanek/nextcloud.nix
@@ -0,0 +1,56 @@
+{ pkgs, lib, ... }:
+let
+  passwordFile = path: toString (pkgs.writeText "password" (lib.strings.fileContents path));
+  inherit (import <niveum/lib>) localAddresses;
+in
+{
+  services.nextcloud = {
+    enable = true;
+    package = pkgs.nextcloud22;
+
+    https = true;
+
+    autoUpdateApps = {
+      enable = true;
+      startAt = "05:00:00";
+    };
+
+    hostName = "cloud.xn--kiern-0qa.de";
+
+    config = {
+      overwriteProtocol = "https";
+
+      dbtype = "pgsql";
+      dbuser = "nextcloud";
+      dbhost = "/run/postgresql"; # nextcloud will add /.s.PGSQL.5432 by itself
+      dbname = "nextcloud";
+      dbpassFile = passwordFile <system-secrets/nextcloud/database>;
+      adminpassFile = passwordFile <system-secrets/nextcloud/admin>;
+      adminuser = "admin";
+      # extraTrustedDomains = [ "toum.r" ];
+    };
+  };
+
+  services.postgresql = {
+    enable = true;
+    ensureDatabases = [ "nextcloud" ];
+    ensureUsers = [
+      {
+        name = "nextcloud";
+        ensurePermissions."DATABASE nextcloud" = "ALL PRIVILEGES";
+      }
+    ];
+  };
+
+
+  services.nginx.virtualHosts."cloud.xn--kiern-0qa.de" = {
+    enableACME = true;
+    forceSSL = true;
+  };
+
+  # Ensure that postgres is running before running the setup
+  systemd.services."nextcloud-setup" = {
+    requires = ["postgresql.service"];
+    after = ["postgresql.service"];
+  };
+}
--- a/systems/makanek/radio/default.nix
+++ b/systems/makanek/radio/default.nix
@@ -0,0 +1,212 @@
+{ lib, pkgs, config, ... }:
+let
+  inherit (import <niveum/lib>) tmpfilesConfig serveHtml;
+
+  radioStore = "/var/lib/radio";
+  htgenPort = 8080;
+  stations = {
+    lyrikline = {
+      streamPort = 8001;
+      mpdPort = 6601;
+      description = ''
+        Weltklang. Welt als ewiges Gedicht, das seine Schallspuren durch Raum und Zeit jagt. Endlose Zufallswiedergabe von <a href="//lyrikline.org">lyrikline</a>. — Listen to the sound of voices and poems permeating linguistic and geographic barriers, 24 hours per day.
+      '';
+    };
+    lyrik = {
+      streamPort = 8002;
+      mpdPort = 6602;
+      description = ''
+        Deutsche Lyrik, die du noch nicht gut genug kennst. Tritt in einen Fluss aus Reim und Maß; keine zwei Mal ist er derselbe.
+      '';
+    };
+  };
+  mpd-add-with-tags = pkgs.writers.writeHaskell "mpd-add-with-tags" {
+    libraries = with pkgs.haskellPackages; [ optparse-generic libmpd ];
+  } ''
+    {-# LANGUAGE DeriveGeneric, OverloadedStrings #-}
+    import Control.Monad (void)
+    import Data.String
+    import Network.MPD
+    import Options.Generic
+
+    data Options = Options { url :: String, artist :: Maybe String, title :: Maybe String }
+      deriving (Generic)
+
+    instance ParseRecord Options
+
+    main :: IO ()
+    main = do
+      options <- getRecord "Add to MPD with tags"
+      void $ withMPD $ do
+        songId <- addId (fromString $ url options) Nothing
+        maybe (pure ()) (addTagId songId Artist . fromString) $ artist options
+        maybe (pure ()) (addTagId songId Title . fromString) $ title options
+  '';
+
+  mpcs = lib.mapAttrs (name: station: pkgs.writers.writeDashBin "mpc-${name}" ''
+    MPD_PORT=${toString station.mpdPort} ${pkgs.mpc_cli}/bin/mpc "$@"
+  '') stations;
+in
+{
+  imports = [ <stockholm/krebs/3modules/htgen.nix> ];
+  nixpkgs.overlays = [
+    (self: super: { htgen = super.callPackage <stockholm/krebs/5pkgs/simple/htgen> {}; })
+  ];
+
+  systemd.tmpfiles.rules = [
+    (tmpfilesConfig {
+      type = "d";
+      path = radioStore;
+      mode = "0755";
+      user = config.users.extraUsers.radio.name;
+      age = "1d";
+    })
+  ];
+
+  users.extraUsers.radio = {
+    isSystemUser = true;
+    group = "radio";
+  };
+  users.groups.radio = {};
+
+  krebs.htgen.radio = {
+    port = htgenPort;
+    user.name = "radio";
+    script = ''. ${pkgs.writers.writeDash "meinskript" ''
+      send200() {
+        printf 'HTTP/1.1 200 OK\r\n'
+        printf 'Content-Type: text/html; charset=UTF-8\r\n'
+        printf 'Connection: close\r\n'
+        printf '\r\n'
+      }
+
+      case "$Method $Request_URI" in
+        "GET /lyrik/status")
+          send200
+          video_id="$(
+            ${mpcs.lyrik}/bin/mpc-lyrik status -f %file% \
+              | head -n1 \
+              | grep -o 'id=[^&]*' \
+              | sed 's/^id=//g'
+          )"
+
+          ${pkgs.youtube-dl}/bin/youtube-dl -j "https://www.youtube.com/watch?v=$video_id" \
+            | ${pkgs.jq}/bin/jq -r '"% [\(.title)](\(.webpage_url))\n\n\(.description)"' \
+            | sed 's/$/  /g' \
+            | ${pkgs.pandoc}/bin/pandoc -s
+
+          exit
+        ;;
+        "GET /lyrikline/status")
+          send200
+
+          hash="$(
+            ${mpcs.lyrikline}/bin/mpc-lyrikline status -f '%file%' \
+              | head -n 1 \
+              | md5sum \
+              | cut -d' ' -f 1
+          )"
+          url="$(cat ${radioStore}/$hash)"
+
+          echo "<html><body style='margin:0'><iframe style='width:100%;height:100%;border:0' src="$url"></iframe></body></html>"
+          exit
+        ;;
+      esac
+    ''}'';
+  };
+
+  containers = lib.mapAttrs (name: station: {
+    autoStart = true;
+    config = {config, pkgs, ...}: {
+      services.mpd = {
+        enable = true;
+        network.port = station.mpdPort;
+        extraConfig = ''
+          log_level "default"
+
+          audio_output {
+            name "${name}"
+            type "httpd"
+            encoder "vorbis"
+            port "${toString station.streamPort}"
+            bitrate "128"
+            format "44100:16:2"
+            always_on "yes"
+            tags "yes"
+          }
+        '';
+      };
+    };
+  }) stations;
+
+  environment.systemPackages = lib.attrValues mpcs;
+
+  systemd.services.lyrikline = {
+    after = [ "container@lyrikline.service" ];
+    wantedBy = [ "container@lyrikline.service" ];
+    startAt = "*:00/5";
+    serviceConfig.User = config.users.extraUsers.radio.name;
+    preStart = "${mpcs.lyrikline}/bin/mpc-lyrikline crop || :";
+    script = ''
+      set -efu
+
+      lyrikline=https://www.lyrikline.org
+      for _ in $(seq 1 10); do
+        random_route="$(${pkgs.curl}/bin/curl -sSL "$lyrikline/index.php/tools/getrandompoem" --data-raw 'lang=de' --compressed | ${pkgs.jq}/bin/jq -r .link)"
+        poem_url="$lyrikline$random_route"
+
+        poem_file="$(
+          ${pkgs.curl}/bin/curl -sSL "$poem_url" \
+            | grep -o 'https://.*\.mp3' \
+            | head -n1
+        )"
+
+        hash="$(echo "$poem_file" | md5sum | cut -d' ' -f 1)"
+        echo "$poem_file ($hash) -> $poem_url"
+        echo "$poem_url" > "${radioStore}/$hash"
+
+        ${mpcs.lyrikline}/bin/mpc-lyrikline add "$poem_file"
+      done
+
+      ${mpcs.lyrikline}/bin/mpc-lyrikline play
+    '';
+  };
+
+  systemd.services.lyrik = {
+    after = [ "container@lyrik.service" ];
+    wantedBy = [ "container@lyrik.service" ];
+    preStart = "${mpcs.lyrik}/bin/mpc-lyrik crop || :";
+    restartIfChanged = true;
+    serviceConfig.User = config.users.extraUsers.radio.name;
+    script =
+      let
+        invidious = "https://invidious.silkky.cloud";
+        videoIds = import <niveum/lib/hot-rotation/lyrik.nix>;
+        streams = lib.concatMapStringsSep "\n" (id: "${invidious}/latest_version?id=${id}&itag=251") videoIds;
+        streamsFile = pkgs.writeText "hotrot" streams;
+      in ''
+        set -efu
+        ${mpcs.lyrik}/bin/mpc-lyrik add < ${toString streamsFile}
+
+        ${mpcs.lyrik}/bin/mpc-lyrik crossfade 5
+        ${mpcs.lyrik}/bin/mpc-lyrik random on
+        ${mpcs.lyrik}/bin/mpc-lyrik repeat on
+        ${mpcs.lyrik}/bin/mpc-lyrik play
+      '';
+  };
+
+
+  services.nginx.virtualHosts."radio.kmein.de" = {
+    enableACME = true;
+    forceSSL = true;
+    locations = lib.mkMerge (
+      [
+        { "/".extraConfig = serveHtml (import ./station-list.nix { inherit pkgs lib stations; }) pkgs; }
+      ] ++ (lib.mapAttrsToList (name: station: {
+        "= /${name}/status".proxyPass = "http://127.0.0.1:${toString htgenPort}";
+        "= /${name}/listen.ogg".proxyPass = "http://127.0.0.1:${toString station.streamPort}";
+        "= /${name}.ogg".return = "301 http://radio.xn--kiern-0qa.de/${name}/listen.ogg"; # legacy
+      }) stations)
+    );
+  };
+}
--- a/systems/makanek/radio/station-list.nix
+++ b/systems/makanek/radio/station-list.nix
@@ -0,0 +1,43 @@
+{ pkgs, lib, stations }:
+let
+  theStations = lib.mapAttrsToList (name: value: value // {name = name;}) stations;
+in
+pkgs.writeText "index.html" ''
+  <!doctype html>
+  <html>
+    <head>
+      <title>radio.kierán</title>
+      <link
+        rel="stylesheet"
+        href="//cdn.rawgit.com/necolas/normalize.css/master/normalize.css"
+      />
+      <link
+        rel="stylesheet"
+        href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.9.2/css/bulma.min.css"
+      />
+      <meta name="viewport" content="width=device-width, initial-scale=1" />
+      <meta charset="utf-8" />
+    </head>
+    <body>
+      <main class="section">
+        <div class="subtitle is-3">Welcome to</div>
+        <h1 class="title is-1">radio.kierán</h1>
+
+        <div class="columns is-multiline">
+          ${lib.concatMapStringsSep "\n" (station: ''
+            <div class="column">
+              <div class="box">
+                <strong class="is-uppercase">${station.name}</strong>
+                (<a href="/${station.name}/status">status</a>, <a href="/${station.name}/listen.ogg">link</a>)
+
+                <p class="has-text-grey">${station.description}</p>
+                <hr/>
+                <audio style="width:100%" controls src="/${station.name}/listen.ogg"/>
+              </div>
+            </div>'') theStations
+          }
+        </div>
+      </main>
+    </body>
+  </html>
+''
--- a/systems/makanek/retiolum-map.nix
+++ b/systems/makanek/retiolum-map.nix
@@ -0,0 +1,69 @@
+{ config, pkgs, lib, ... }:
+let
+  network = "retiolum";
+
+  stateDirectory = "retiolum-map";
+
+  geo-ip-database = "${lib.head config.services.geoipupdate.settings.EditionIDs}.mmdb";
+  geo-ip-database-path = "${config.services.geoipupdate.settings.DatabaseDirectory}/${geo-ip-database}";
+
+  tinc-graph-source = pkgs.fetchFromGitHub {
+    owner = "kmein";
+    repo = "tinc-graph";
+    rev = "1b9c69ffdbdab82d30a23e44f739feea63d94163";
+    sha256 = "0bk4x9l399f80lmz4asx7s5fcsgrfdb6xyymbq884621x4z3d2vp";
+  };
+  tinc-graph = pkgs.callPackage tinc-graph-source {};
+in
+{
+  systemd.services.retiolum-index = {
+    description = "Retiolum indexing service";
+    wants = [ "tinc.${network}.service" ];
+    script = ''
+      ${tinc-graph}/bin/tinc-graph --geoip-file ${geo-ip-database-path} --network ${network} \
+        | ${pkgs.coreutils}/bin/tee network.json \
+        | ${tinc-graph}/bin/tinc-statistics > statistics.json
+
+      cp ${tinc-graph}/static/map.html map.html
+      cp ${tinc-graph}/static/map.html index.html
+      cp ${tinc-graph}/static/graph.html graph.html
+    '';
+    startAt = "hourly";
+    path = [ pkgs.coreutils pkgs.jq pkgs.tinc_pre ];
+    serviceConfig = {
+      Type = "oneshot";
+      User = "root";
+      StateDirectory = stateDirectory;
+      WorkingDirectory = "/var/lib/${stateDirectory}";
+    };
+  };
+
+  services.geoipupdate = {
+    enable = true;
+    settings = {
+      AccountID = 608777;
+      LicenseKey = toString <system-secrets/maxmind/license.key>;
+      EditionIDs = [ "GeoLite2-City" ];
+    };
+  };
+
+  services.nginx = {
+    enable = true;
+    recommendedGzipSettings = true;
+    recommendedOptimisation = true;
+    recommendedProxySettings = true;
+    recommendedTlsSettings = true;
+    virtualHosts."graph.r".locations."/".root = "/var/lib/${stateDirectory}";
+  };
+
+  systemd.services.geoip-share = {
+    after = [ "geoipupdate.service" ];
+    script = let
+      cyberlocker-tools = pkgs.callPackage <stockholm/krebs/5pkgs/simple/cyberlocker-tools> {};
+    in "${cyberlocker-tools}/bin/cput ${geo-ip-database} < ${geo-ip-database-path}";
+    serviceConfig = {
+      Type = "oneshot";
+      DynamicUser = true;
+    };
+  };
+}
--- a/systems/makanek/tarot.nix
+++ b/systems/makanek/tarot.nix
@@ -0,0 +1,61 @@
+{ config, pkgs, lib, ... }:
+let
+  tarotPort = 7407;
+  tarotFiles = pkgs.fetchzip {
+    url = "https://c.krebsco.de/tarot.zip";
+    sha256 = "0jl5vdwlj17pqp94yj02xgsb1gyvs9i08m83kac0jdnhfjl2f75a";
+    stripRoot = false;
+  };
+  tarotKey = builtins.fetchurl {
+    url = "http://c.krebsco.de/tarot.pdf";
+    sha256 = "1n2m53kjg2vj9dbr70b9jrsbqwdfrcb48l4wswn21549fi24g6dx";
+  };
+in
+{
+  krebs.htgen.tarot = {
+    port = tarotPort;
+    user.name = "radio";
+    script = ''. ${pkgs.writers.writeDash "tarot" ''
+      case "$Method $Request_URI" in
+        "GET /")
+          if item=$(${pkgs.findutils}/bin/find ${toString tarotFiles} -type f | ${pkgs.coreutils}/bin/shuf -n1); then
+            card=$(mktemp --tmpdir tarot.XXX)
+            trap 'rm $card' EXIT
+            reverse=$(${pkgs.coreutils}/bin/shuf -i0-1 -n1)
+            if [ "$reverse" -eq 1 ]; then
+              ${pkgs.imagemagick}/bin/convert -rotate 180 "$item" "$card"
+            else
+              ${pkgs.coreutils}/bin/cp "$item" "$card"
+            fi
+            printf 'HTTP/1.1 200 OK\r\n'
+            printf 'Content-Type: %s\r\n' "$(${pkgs.file}/bin/file -ib "$card")"
+            printf 'Server: %s\r\n' "$Server"
+            printf 'Connection: close\r\n'
+            printf 'Content-Length: %d\r\n' $(${pkgs.coreutils}/bin/wc -c < "$card")
+            printf '\r\n'
+            cat "$card"
+            exit
+          fi
+        ;;
+      esac
+    ''}'';
+  };
+
+  services.nginx.virtualHosts."tarot.kmein.de" = {
+    enableACME = true;
+    forceSSL = true;
+    locations = {
+      "/".proxyPass = "http://127.0.0.1:${toString tarotPort}";
+      "/files/" = {
+        root = pkgs.linkFarm "tarot" [
+          { name = "files/key.pdf"; path = tarotKey; }
+          { name = "files/cards"; path = tarotFiles; }
+        ];
+        extraConfig = ''
+          autoindex on;
+          charset UTF-8;
+        '';
+      };
+    };
+  };
+}
--- a/systems/makanek/urlwatch.nix
+++ b/systems/makanek/urlwatch.nix
@@ -0,0 +1,159 @@
+{ pkgs, config, lib, ... }:
+let
+  inherit (import <niveum/lib>) kieran;
+
+  urlwatchDir = "/var/lib/urlwatch";
+
+  urlsFile = pkgs.writeText "urls" (builtins.concatStringsSep "\n---\n" (map builtins.toJSON urls));
+
+  urls = [
+    {
+      name = "Corona-Verordnung";
+      url = "https://www.berlin.de/corona/massnahmen/verordnung/";
+      filter = [ { css = "[role=main]"; } "html2text" "strip" ];
+    }
+    {
+      name = "HU Semester";
+      url = "https://agnes.hu-berlin.de/lupo/rds?state=change&type=6&moduleParameter=semesterSelect&nextdir=change&next=SearchSelect.vm&subdir=applications&targettype=7&targetstate=change&getglobal=semester";
+      filter = [ { css = "fieldset"; } "html2text" "strip" ];
+    }
+    {
+      name = "Lammla 2021";
+      url = "http://lammla.info/index.php?reihe=30";
+      filter = [ "html2text" "strip" ];
+    }
+    {
+      name = "Tatort";
+      url = "https://www.daserste.de/unterhaltung/krimi/tatort/vorschau/index.html";
+      filter = [
+        "html2text"
+        "strip"
+        {
+          shellpipe = ''
+            ${pkgs.gnused}/bin/sed 's/&#32;/ /g;s/))/&\n/g;s/ \+/ /g'
+          '';
+        }
+      ];
+    }
+    {
+      name = "Kratylos";
+      url = "https://kratylos.reichert-online.org/current_issue/KRATYLOS";
+      filter = [ { element-by-id = "content"; } "html2text" "strip" ];
+    }
+    {
+      name = "Zeno Free E-Books";
+      url = "http://www.zeno.org/Lesesaal/M/E-Books";
+      filter = [ { element-by-class = "zenoCOMain"; } "html2text" "strip" ];
+    }
+    {
+      name = "Indogermanische Forschungen";
+      url = "https://www.degruyter.com/journal/key/INDO/html";
+      filter = [ { element-by-id = "latestIssue"; } "html2text" "strip" ];
+    }
+    {
+      name = "IG Neuigkeiten";
+      url = "https://www.indogermanistik.org/aktuelles/neuigkeiten.html";
+      filter = [ { css = "[itemprop=articleBody]"; } "html2text" "strip" ];
+    }
+    {
+      name = "IG Tagungen";
+      url = "https://www.indogermanistik.org/tagungen/tagungen-der-ig.html";
+      filter = [ { css = "[itemprop=articleBody]"; } "html2text" "strip" ];
+    }
+    {
+      name = "Christian-Metz-Blamage";
+      url = "https://www.deutschlandfunk.de/meine-nacht-schlaeft-nicht-pflanze-mich-nicht-in-dein-herz.700.de.html?dram:article_id=486475";
+      filter = [ { element-by-class = "dlf-articledetail"; } "html2text" "strip" ];
+    }
+    {
+      name = "fxght.or.flxght";
+      url = "https://api.tellonym.me/profiles/name/fxght.or.flxght?limit=20";
+      headers.tellonym-client = "web:0.52.0";
+      filter = [
+        {
+          shellpipe = ''
+            ${pkgs.jq}/bin/jq '.answers | map({
+              question: .tell,
+              answer: .answer,
+              date: .createdAt,
+              media: .media | map(.url)
+            })'
+          '';
+        }
+      ];
+    }
+  ];
+
+  configFile = (pkgs.formats.yaml {}).generate "urlwatch.yaml" {
+    display = {
+      error = true;
+      new = true;
+      unchanged = false;
+    };
+    report = {
+      email = {
+        enabled = true;
+        from = "2210@cock.li";
+        html = false;
+        method = "smtp";
+        smtp = {
+          host = "mail.cock.li";
+          port = 587;
+          starttls = true;
+          auth = true;
+          insecure_password = lib.strings.fileContents <secrets/mail/cock>;
+        };
+        subject = "{count} changes: {jobs}";
+        to = kieran.email;
+      };
+      telegram = {
+        enabled = true;
+        bot_token = lib.strings.fileContents <system-secrets/telegram/kmein.token>;
+        chat_id = "-1001504043752";
+      };
+      html.diff = "unified";
+      stdout = {
+        color = true;
+        enabled = true;
+      };
+      text.footer = false;
+      # telegram = {
+      #   enabled = false;
+      #   bot_token = lib.strings.fileContents <system-secrets/telegram/kmein.token>;
+      #   chat_id = [ "18980945" ];
+      # };
+    };
+  };
+  urlwatch = pkgs.urlwatch.overrideAttrs (attrs: {
+    patches = [ <niveum/packages/urlwatch-insecure.patch> ];
+  });
+in
+{
+  users.extraUsers.urlwatch = {
+    home = urlwatchDir;
+    createHome = true;
+    isSystemUser = true;
+    group = "urlwatch";
+  };
+
+  users.groups.urlwatch = {};
+
+  systemd.services.urlwatch = {
+    enable = true;
+    startAt = "*-*-* 05:00:00";
+    script = ''
+      ${urlwatch}/bin/urlwatch \
+        --config=${lib.escapeShellArg configFile} \
+        --urls=${lib.escapeShellArg urlsFile}
+    '';
+    serviceConfig = {
+      User = config.users.extraUsers.urlwatch.name;
+      Group = config.users.groups.urlwatch.name;
+      WorkingDirectory = config.users.extraUsers.urlwatch.home;
+      PermissionsStartOnly = "true";
+      PrivateTmp = "true";
+      SyslogIdentifier = "urlwatch";
+      Type = "oneshot";
+    };
+  };
+}
--- a/systems/makanek/weechat.nix
+++ b/systems/makanek/weechat.nix
@@ -0,0 +1,123 @@
+{ lib, pkgs, ... }:
+let
+  inherit (import <niveum/lib>) kieran;
+  relayPassword = lib.fileContents <system-secrets/weechat/relay>;
+in {
+  systemd.services.weechat =
+  let
+    tmux = pkgs.writers.writeDash "tmux" ''
+      exec ${pkgs.tmux}/bin/tmux -f ${pkgs.writeText "tmux.conf" ''
+        set-option -g prefix `
+        unbind-key C-b
+        bind ` send-prefix
+
+        set-option -g status off
+        set-option -g default-terminal screen-256color
+
+        #use session instead of windows
+        bind-key c new-session
+        bind-key p switch-client -p
+        bind-key n switch-client -n
+        bind-key C-s switch-client -l
+      ''} "$@"
+    '';
+    weechat = pkgs.weechat.override {
+      configure = { ... }: {
+        scripts = [ pkgs.weechatScripts.weechat-autosort pkgs.weechatScripts.colorize_nicks pkgs.weechatScripts.weechat-matrix ];
+        init = let
+          coolColors = lib.lists.subtractLists (lib.range 52 69 ++ lib.range 231 248) (lib.range 31 254);
+          nick = "kmein";
+        in ''
+          /mouse enable
+          /set irc.server_default.nicks "${nick}"
+          /set irc.server_default.msg_part "tschö mit ö"
+          /set irc.server_default.msg_quit "ciao kakao"
+          /set irc.server_default.msg_kick "warum machst du diese?"
+          /set irc.server_default.realname "${kieran.name}"
+
+          /set irc.look.color_nicks_in_nicklist "on"
+          /set weechat.color.chat_nick_colors "${lib.concatMapStringsSep "," toString coolColors}"
+
+          /server add hackint irc.hackint.org/6697 -ipv6 -ssl
+          /server add libera irc.libera.chat/6697 -ssl
+          /server add oftc irc.oftc.net/6697 -ssl -ipv6
+          /server add retiolum irc.r
+          /server add news news.r
+          /matrix server add nibbana nibbana.jp
+
+          /alias add mod /quote omode $channel +o $nick
+
+          /relay add weechat 9000
+          /set relay.network.password ${relayPassword}
+
+          /set matrix.server.nibbana.username ${nick}
+          /set matrix.server.nibbana.password "${lib.strings.fileContents <system-secrets/matrix/nibbana>}"
+
+          /set irc.server.oftc.command /msg nickserv IDENTIFY ${lib.strings.fileContents <system-secrets/irc/oftc>};/msg nickserv SET CLOAK ON
+          /set irc.server.oftc.autojoin "#osm,#osm-de,#home-manager"
+
+          /set irc.server.hackint.autojoin "#krebs,#nixos,#the_playlist"
+          /set irc.server.hackint.sasl_mechanism plain
+          /set irc.server.hackint.sasl_username ${nick}
+          /set irc.server.hackint.sasl_password ${lib.strings.fileContents <system-secrets/irc/hackint>}
+
+          /set irc.server.libera.autojoin "#flipdot,#haskell,#nixos,#fysi,#binaergewitter"
+          /set irc.server.libera.sasl_mechanism plain
+          /set irc.server.libera.sasl_username ${nick}
+          /set irc.server.libera.sasl_password ${lib.strings.fileContents <system-secrets/irc/libera>}
+
+          /set irc.server.retiolum.autojoin "#xxx,#brockman,#flix,#autowifi"
+          /set irc.server.retiolum.command "/oper admin aidsballs;/msg nickserv always-on true;/msg nickserv autoreplay-missed on;/msg nickserv auto-away"
+          /set irc.server.retiolum.sasl_mechanism plain
+          /set irc.server.retiolum.sasl_username ${nick}
+          /set irc.server.retiolum.sasl_password ${lib.strings.fileContents <system-secrets/irc/retiolum>}
+
+          /set irc.server.news.autojoin "#cook,#drachengame,#oepnv,#kmeinung,#memes"
+          /set irc.server.news.command "/oper aids balls"
+          /set logger.level.irc.news 0
+
+          /filter addreplace zerocovid * * [kc]orona|💉|🤒|😷|[kc]ovid|virus|lockdown|va[kc][sc]in|mutante|mutation|impf|pandemi|κορ[ωο]ν[αο]ϊό|корона|expert|infe[ck]t|infizi|in[cz]iden[cz]|sars-cov|drosten|virolog|lauterbach|delta|omi[ck]ron|epidemi|booster|r-wert
+          /filter addreplace joinquit * irc_join,irc_part,irc_quit,irc_nick *
+          /filter addreplace playlist_topic irc.*.#the_playlist irc_topic *
+          /filter addreplace brockman_notice irc.news.* irc_notice *
+
+          /set irc.look.server_buffer independent
+
+          /connect libera
+          /connect oftc
+          /connect hackint
+          /connect retiolum
+          /connect news
+          /matrix connect nibbana
+        '';
+      };
+    };
+  in {
+    description = "Weechat bouncer";
+    after = [ "network.target" ];
+    wantedBy = [ "multi-user.target" ];
+    restartIfChanged = true;
+    path = [ pkgs.alacritty.terminfo ];
+    environment.WEECHAT_HOME = "/var/lib/weechat";
+    script = "${tmux} -2 new-session -d -s IM ${weechat}/bin/weechat";
+    preStop = "${tmux} kill-session -t IM";
+    serviceConfig = {
+      User = "weechat";
+      RemainAfterExit = true;
+      Type = "oneshot";
+    };
+  };
+
+  users.groups.weechat = {};
+  users.extraUsers.weechat = {
+    useDefaultShell = true;
+    openssh.authorizedKeys.keys = kieran.sshKeys pkgs ++ [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC+KVDmYYH7mA8v81e9O3swXm3ZVYY9t4HP65ud61uXy weechat_android@heym"
+    ];
+    createHome = true;
+    group = "weechat";
+    home = "/var/lib/weechat";
+    isSystemUser = true;
+    packages = [ pkgs.tmux ];
+  };
+}
--- a/systems/manakish/configuration.nix
+++ b/systems/manakish/configuration.nix
@@ -1,7 +1,3 @@
-# Edit this configuration file to define what should be installed on
-# your system.  Help is available in the configuration.nix(5) man page
-# and in the NixOS manual (accessible by running ‘nixos-help’).
-
 { config, pkgs, ... }:
 let
  inherit (import <niveum/lib>) retiolumAddresses;
@@ -9,25 +5,10 @@ in
 {
  imports = [ # Include the results of the hardware scan.
    ./hardware-configuration.nix
+    ./hdmi.nix
    <niveum/configs/default.nix>
    <niveum/configs/battery.nix>
    <niveum/configs/wpa_supplicant.nix>
-    {
-      programs.steam.enable = true;
-      nixpkgs.config.steam.java = true;
-    }
-    {
-      services.xserver.displayManager.sessionCommands =
-      let
-        intern = "LVDS-1";
-        extern = "HDMI-1";
-        pulseaudioCard = "alsa_card.pci-0000_00_1b.0";
-        pulseaudioProfile = "output:hdmi-stereo+input:analog-stereo";
-      in toString (pkgs.writers.writeDash "hdmi-on" ''
-        ${pkgs.xorg.xrandr}/bin/xrandr --output ${intern} --primary --auto --output ${extern} --above ${intern} --auto
-        ${pkgs.pulseaudio}/bin/pactl set-card-profile ${pulseaudioCard} ${pulseaudioProfile}
-      '');
-    }
  ];

  niveum = {
@@ -36,21 +17,17 @@ in
    promptColours.success = "green";
  };

-  networking.useDHCP = false;
-  networking.interfaces = {
-    enp0s25.useDHCP = true;
-    wlp3s0.useDHCP = true;
-    wwp0s20u4i6.useDHCP = true;
+  networking = {
+    useDHCP = false;
+    interfaces = {
+      enp0s25.useDHCP = true;
+      wlp3s0.useDHCP = true;
+      wwp0s20u4i6.useDHCP = true;
+    };
+    wireless.interfaces = [ "wlp3s0" ];
+    retiolum = retiolumAddresses.manakish;
+    networking.hostName = "manakish";
  };
-  networking.wireless.interfaces = [ "wlp3s0" ];
-
-  environment.systemPackages = with pkgs; [
-    git vim
-  ];
-
-  networking.retiolum = retiolumAddresses.manakish;
-
-  networking.hostName = "manakish";

  system.stateVersion = "20.09"; # Did you read the comment?
 }
--- a/systems/manakish/hardware-configuration.nix
+++ b/systems/manakish/hardware-configuration.nix
@@ -1,40 +1,37 @@
-# Do not modify this file!  It was generated by ‘nixos-generate-config’
-# and may be overwritten by future invocations.  Please make changes
-# to /etc/nixos/configuration.nix instead.
 { config, lib, pkgs, modulesPath, ... }:
-
 {
-  imports =
-    [ (modulesPath + "/installer/scan/not-detected.nix")
-    ];
+  imports = [ (modulesPath + "/installer/scan/not-detected.nix") ];

-  boot.initrd.availableKernelModules = [ "xhci_pci" "ehci_pci" "ahci" "usb_storage" "sd_mod" "sdhci_pci" ];
-  boot.initrd.kernelModules = [ ];
-  boot.kernelModules = [ "kvm-intel" ];
-  boot.extraModulePackages = [ ];
+  boot = {
+    initrd = {
+      availableKernelModules = [ "xhci_pci" "ehci_pci" "ahci" "usb_storage" "sd_mod" "sdhci_pci" ];
+      kernelModules = [];
+      luks.devices."crypted".device = "/dev/disk/by-uuid/493cb228-c292-4f71-9f38-dcb3e96dec47";
+    };
+    kernelModules = [ "kvm-intel" ];
+    extraModulePackages = [ ];
+    loader.grub = {
+      enable = true;
+      version = 2;
+      efiSupport = true;
+      efiInstallAsRemovable = true;
+      device = "/dev/sda";
+    };
+  };

-  fileSystems."/" =
-    { device = "/dev/disk/by-uuid/ff47c479-e6d4-4cd6-93f1-9685f4018bc6";
+  fileSystems = {
+    "/" = {
+      device = "/dev/disk/by-uuid/ff47c479-e6d4-4cd6-93f1-9685f4018bc6";
      fsType = "ext4";
    };
-
-  boot.initrd.luks.devices."crypted".device = "/dev/disk/by-uuid/493cb228-c292-4f71-9f38-dcb3e96dec47";
-
-  fileSystems."/boot" =
-    { device = "/dev/disk/by-uuid/D4AC-91B0";
+    "/boot" = {
+      device = "/dev/disk/by-uuid/D4AC-91B0";
      fsType = "vfat";
    };
+  };

  swapDevices = [ ];
  zramSwap.enable = true;

-  boot.loader.grub = {
-    enable = true;
-    version = 2;
-    efiSupport = true;
-    efiInstallAsRemovable = true;
-    device = "/dev/sda";
-  };
-
  powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
 }
--- a/systems/manakish/hdmi.nix
+++ b/systems/manakish/hdmi.nix
@@ -0,0 +1,13 @@
+{ pkgs, ... }:
+{
+  services.xserver.displayManager.sessionCommands =
+  let
+    intern = "LVDS-1";
+    extern = "HDMI-1";
+    pulseaudioCard = "alsa_card.pci-0000_00_1b.0";
+    pulseaudioProfile = "output:hdmi-stereo+input:analog-stereo";
+  in toString (pkgs.writers.writeDash "hdmi-on" ''
+    ${pkgs.xorg.xrandr}/bin/xrandr --output ${intern} --primary --auto --output ${extern} --above ${intern} --auto
+    ${pkgs.pulseaudio}/bin/pactl set-card-profile ${pulseaudioCard} ${pulseaudioProfile}
+  '');
+}
--- a/systems/toum/configuration.nix
+++ b/systems/toum/configuration.nix
@@ -1,46 +0,0 @@
-{ config, pkgs, lib, ... }:
-let
-  inherit (import <niveum/lib>) kieran;
-in {
-  imports = [
-    ./hardware-configuration.nix
-    # <niveum/configs/hass>
-    <niveum/configs/distrobump.nix>
-    <niveum/configs/spacetime.nix>
-    <niveum/configs/sshd.nix>
-    <niveum/configs/save-space.nix>
-    <niveum/configs/tmux.nix>
-    <niveum/configs/version.nix>
-    <niveum/configs/traadfri.nix>
-    <niveum/configs/wpa_supplicant.nix>
-    <niveum/modules/retiolum.nix>
-  ];
-
-  nix.nixPath = [ "/var/src" ];
-
-  boot.loader.grub.enable = false;
-  boot.loader.generic-extlinux-compatible.enable = true;
-
-  networking.hostName = "toum";
-
-  environment.variables.TERM = "linux";
-
-  environment.systemPackages = with pkgs; [
-    git vim htop wget reptyr
-    libraspberrypi
-  ];
-
-  users.mutableUsers = false;
-
-  networking.retiolum = {
-    ipv4 = "10.243.2.3";
-    ipv6 = "42:0:3c46:56af:d12b:affd:8390:df22";
-  };
-
-  environment.etc."tinc/retiolum/rsa_key.priv" = {
-    text = builtins.readFile <system-secrets/retiolum.key>;
-    mode = "400";
-  };
-
-  system.stateVersion = "20.03";
-}
--- a/systems/toum/hardware-configuration.nix
+++ b/systems/toum/hardware-configuration.nix
@@ -1,16 +0,0 @@
-{ config, pkgs, ... }: {
-  boot.kernelPackages = pkgs.linuxPackages; # https://github.com/NixOS/nixpkgs/issues/97064#issuecomment-708986741
-  boot.kernelParams = [ "cma=32M" ];
-
-  fileSystems = {
-    "/" = {
-      device = "/dev/disk/by-label/NIXOS_SD";
-      fsType = "ext4";
-    };
-  };
-
-  swapDevices = [{
-    device = "/swapfile";
-    size = 1024;
-  }];
-}
--- a/systems/zaatar/bvg.nix
+++ b/systems/zaatar/bvg.nix
@@ -0,0 +1,57 @@
+{ pkgs, lib, ... }:
+let
+  inherit (import <niveum/lib>) serveHtml;
+  stations = [
+    900068204 # A/M
+    900068302 # KAS
+    900068203 # B-P
+  ];
+  fahrplan = pkgs.writeText "fahrplan.html" ''
+    <!DOCTYPE html>
+    <title>Fahrplan</title>
+    <link
+      rel="icon"
+      type="image/x-icon"
+      href="https://mobil.bvg.de/Fahrinfo/img/ua_xhtml/logo.gif"
+    />
+    <style>
+      body {
+        margin: 0;
+        --bvg-yellow: #f0d722;
+      }
+      #fahrplan {
+        display: flex;
+        height: 100vh;
+        width: 100%;
+        flex-direction: row;
+      }
+      #fahrplan iframe {
+        flex-grow: 1;
+        border: none;
+      }
+      #fahrplan iframe + iframe {
+        border-left: 2px solid var(--bvg-yellow);
+      }
+    </style>
+    <body>
+      <div id="fahrplan">
+        ${lib.concatMapStrings (station: ''
+          <iframe scrolling="no" src="https://mobil.bvg.de/Fahrinfo/bin/stboard.bin/dox?ld=0.1&input=${toString station}&boardType=depRT&start=yes"></iframe>
+        '') stations}
+      </div>
+    </body>
+  '';
+in
+{
+  services.nginx = {
+    enable = true;
+    recommendedGzipSettings = true;
+    recommendedOptimisation = true;
+    recommendedProxySettings = true;
+    recommendedTlsSettings = true;
+  };
+
+  services.nginx.virtualHosts."bvg.kmein.r" = {
+    locations."/".extraConfig = serveHtml fahrplan pkgs;
+  };
+}
--- a/systems/zaatar/configuration.nix
+++ b/systems/zaatar/configuration.nix
@@ -4,100 +4,24 @@ let
 in
 {
  imports = [
+    ./bvg.nix
+    ./gaslight.nix
    ./hardware-configuration.nix
+    ./kiosk.nix
+    ./moodle-dl-meinhark.nix
+    ./pulseaudio.nix
+    ./tuna.nix
    <home-manager/nixos>
-    <niveum/configs/wpa_supplicant.nix>
    <niveum/configs/keyboard.nix>
-    <niveum/modules/retiolum.nix>
-    <niveum/configs/spacetime.nix>
+    <niveum/configs/monitoring.nix>
    <niveum/configs/nix.nix>
-    <niveum/configs/tuna.nix>
-    <niveum/configs/sshd.nix>
-    <niveum/configs/version.nix>
-    <niveum/configs/gaslight.nix>
    <niveum/configs/printing.nix>
+    <niveum/configs/spacetime.nix>
+    <niveum/configs/sshd.nix>
    <niveum/configs/traadfri.nix>
-    <niveum/configs/bvg.nix>
-    <niveum/configs/moodle-dl/meinhark.nix>
-    <niveum/configs/monitoring/push.nix>
-    {
-      nixpkgs.config.allowUnfree = true;
-    }
-    {
-      sound.enable = true;
-
-      hardware.pulseaudio = {
-        enable = true;
-        systemWide = true;
-        tcp = {
-          enable = true;
-          anonymousClients.allowedIpRanges = [ "127.0.0.1" "10.243.2.0/24" "192.168.0.0/16" ];
-        };
-        zeroconf.publish.enable = true;
-      };
-      networking.firewall.allowedTCPPorts = [ 4713 ];
-    }
-    {
-      environment.systemPackages =
-      let
-        worldradio = pkgs.callPackage <niveum/packages/worldradio.nix> {};
-      in [
-        (pkgs.writers.writeDashBin "mpv" ''
-          ${pkgs.mpv}/bin/mpv --no-video "$@"
-        '')
-        (pkgs.writers.writeDashBin "worldradio" ''
-          shuf ${worldradio} | ${pkgs.findutils}/bin/xargs ${pkgs.mpv}/bin/mpv --no-video
-        '')
-      ];
-    }
-    {
-      users.extraUsers.kiosk = {
-        isNormalUser = true;
-        password = "";
-        extraGroups = [ "audio" ];
-      };
-      services.cage = {
-        enable = true;
-        user = config.users.extraUsers.kiosk.name;
-        extraArguments = [ "-s" ]; # allow vt switching
-        program =
-        let startUrls = [ "https://open.spotify.com" "https://youtube.com" "http://bvg.kmein.r" ];
-        in pkgs.writers.writeDash "kiosk-browser" ''
-          while true; do
-            ${pkgs.chromium}/bin/chromium \
-              --no-first-run --no-message-box --noerrdialogs \
-              --default-browser --no-default-browser-check \
-              --start-maximized ${lib.escapeShellArgs startUrls}
-            sleep 0.5
-          done
-        '';
-      };
-      systemd.services.cage-tty1.environment.XKB_DEFAULT_LAYOUT = "de";
-      programs.chromium = {
-        enable = true;
-        extensions = [
-          "cjpalhdlnbpafiamejdnhcphjbkeiagm" # uBlock Origin
-        ];
-      };
-    }
-    {
-      environment.systemPackages = [ pkgs.tmux ];
-      systemd.services.turntables = {
-        description = "music controller session";
-        after = [ "network.target" ];
-        wantedBy = [ "multi-user.target" ];
-        path = [ pkgs.alacritty.terminfo ];
-        script = ''
-          ${pkgs.tmux}/bin/tmux -2 new-session -d -s turntables ${pkgs.alsaUtils}/bin/alsamixer \; new-window
-        '';
-        preStop = "${pkgs.tmux}/bin/tmux kill-session -t turntables";
-        serviceConfig = {
-          User = "root";
-          RemainAfterExit = true;
-          Type = "oneshot";
-        };
-      };
-    }
+    <niveum/configs/version.nix>
+    <niveum/configs/wpa_supplicant.nix>
+    <niveum/modules/retiolum.nix>
  ];

  nix.nixPath = [ "/var/src" ];
@@ -110,19 +34,25 @@ in

  services.illum.enable = true;

-  environment.systemPackages = with pkgs; [ git vim htop ncmpcpp ];
+  environment.systemPackages =
+  let
+    worldradio = pkgs.callPackage <niveum/packages/worldradio.nix> {};
+  in [
+    (pkgs.writers.writeDashBin "mpv" ''${pkgs.mpv}/bin/mpv --no-video "$@"'')
+    (pkgs.writers.writeDashBin "worldradio" ''
+      shuf ${worldradio} | ${pkgs.findutils}/bin/xargs ${pkgs.mpv}/bin/mpv --no-video
+    '')
+    pkgs.git
+    pkgs.vim
+    pkgs.htop
+    pkgs.ncmpcpp
+  ];

-  boot.loader.systemd-boot = {
-    enable = true;
-    configurationLimit = 5;
+  networking = {
+    hostName = "zaatar";
+    wireless.interfaces = [ "wlp2s0" ];
+    retiolum = retiolumAddresses.zaatar;
  };

-  boot.loader.efi.canTouchEfiVariables = true;
-
-  networking.hostName = "zaatar";
-  networking.wireless.interfaces = [ "wlp2s0" ];
-
-  networking.retiolum = retiolumAddresses.zaatar;
-
  system.stateVersion = "20.09";
 }
--- a/systems/zaatar/gaslight.nix
+++ b/systems/zaatar/gaslight.nix
@@ -0,0 +1,25 @@
+{ pkgs, ... }:
+{
+  environment.systemPackages = [
+    (pkgs.writers.writeDashBin "gaslight-stream" ''
+      ${pkgs.ffmpeg}/bin/ffmpeg -r 14 -s 640x480 -f video4linux2 -i /dev/video0 -f alsa -i default -c:v libx264 -preset ultrafast -c:a aac -f avi -
+    '')
+    (pkgs.writers.writeDashBin "gaslight-say" ''
+      voices="de
+      de+whisper"
+
+      echo "$@" | ${pkgs.espeak}/bin/espeak -v "$(echo "$voices" | ${pkgs.coreutils}/bin/shuf -n1)"
+    '')
+    (pkgs.writers.writeDashBin "gaslight-play" ''
+      set -o noglob
+      ${pkgs.mpv}/bin/mpv --no-video "$1"
+    '')
+  ];
+}
+
+/*
+
+ssh machine gaslight-stream | mpv -
+ssh machine gaslight-say "blablabla"
+
+*/
--- a/systems/zaatar/hardware-configuration.nix
+++ b/systems/zaatar/hardware-configuration.nix
@@ -1,23 +1,32 @@
-{ config, lib, pkgs, ... }: {
+{ config, lib, pkgs, ... }:
+{
  imports = [ <nixpkgs/nixos/modules/installer/scan/not-detected.nix> ];

-  boot.initrd.availableKernelModules =
-    [ "ahci" "xhci_pci" "usb_storage" "sd_mod" "sdhci_acpi" "rtsx_usb_sdmmc" ];
-  boot.kernelModules = [ "kvm-intel" ];
-  boot.extraModulePackages = [ ];
-
-  fileSystems."/" = {
-    device = "/dev/disk/by-label/nixos";
-    fsType = "ext4";
+  boot = {
+    initrd.availableKernelModules = [ "ahci" "xhci_pci" "usb_storage" "sd_mod" "sdhci_acpi" "rtsx_usb_sdmmc" ];
+    kernelModules = [ "kvm-intel" ];
+    extraModulePackages = [ ];
+    loader = {
+      systemd-boot = {
+        enable = true;
+        configurationLimit = 5;
+      };
+      efi.canTouchEfiVariables = true;
+    };
  };

-  fileSystems."/boot" = {
-    device = "/dev/disk/by-label/boot";
-    fsType = "vfat";
+  fileSystems = {
+    "/" = {
+      device = "/dev/disk/by-label/nixos";
+      fsType = "ext4";
+    };
+    "/boot" = {
+      device = "/dev/disk/by-label/boot";
+      fsType = "vfat";
+    };
  };

-  swapDevices =
-    [{ device = "/dev/disk/by-label/swap"; }];
+  swapDevices = [{ device = "/dev/disk/by-label/swap"; }];

  nix.maxJobs = lib.mkDefault 4;
  powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
--- a/systems/zaatar/kiosk.nix
+++ b/systems/zaatar/kiosk.nix
@@ -0,0 +1,31 @@
+{ config, pkgs, lib, ... }:
+{
+  users.extraUsers.kiosk = {
+    isNormalUser = true;
+    password = "";
+    extraGroups = [ "audio" ];
+  };
+  services.cage = {
+    enable = true;
+    user = config.users.extraUsers.kiosk.name;
+    extraArguments = [ "-s" ]; # allow vt switching
+    program =
+    let startUrls = [ "https://open.spotify.com" "https://youtube.com" "http://bvg.kmein.r" ];
+    in pkgs.writers.writeDash "kiosk-browser" ''
+      while true; do
+        ${pkgs.chromium}/bin/chromium \
+          --no-first-run --no-message-box --noerrdialogs \
+          --default-browser --no-default-browser-check \
+          --start-maximized ${lib.escapeShellArgs startUrls}
+        sleep 0.5
+      done
+    '';
+  };
+  systemd.services.cage-tty1.environment.XKB_DEFAULT_LAYOUT = "de";
+  programs.chromium = {
+    enable = true;
+    extensions = [
+      "cjpalhdlnbpafiamejdnhcphjbkeiagm" # uBlock Origin
+    ];
+  };
+}
--- a/systems/zaatar/moodle-dl-meinhark.nix
+++ b/systems/zaatar/moodle-dl-meinhark.nix
@@ -0,0 +1,107 @@
+{ config, pkgs, lib, ... }:
+let
+  moodle-dl-package = pkgs.moodle-dl.overrideAttrs (old: old // {
+    patches = [ <niveum/packages/moodle-dl/telegram-format.patch> ];
+  });
+in
+{
+  imports = [ <niveum/modules/moodle-dl.nix> ];
+
+  services.moodle-dl = {
+    enable = true;
+    startAt = "hourly";
+    package = moodle-dl-package;
+    settings = {
+      telegram = {
+        token = lib.strings.fileContents <system-secrets/telegram/moodle-dl.token>;
+        chat_id = "18980945";
+        send_error_msg = false;
+      };
+      token = lib.strings.fileContents <system-secrets/moodle.token>;
+      moodle_domain = "moodle.hu-berlin.de";
+      moodle_path = "/";
+      download_course_ids = [
+        # WS 2020
+        99881 # Dialektologie
+        100183 # Onomastik
+        100353 # Sanskrit I
+        100692 # Sanskrit Tutorium
+        99832 # Germanisch
+        99823 # Gotisch
+        99813 # Altalbanisch
+        98681 # Geistliche Lyrik von Luther bis Lehnert
+        99667 # Antike Mythologie
+        # 52365 # FSR KlassPhil
+
+        # SS 2021
+        104850 # Metrik
+        103274 # Marc Aurel
+        102909 # Sanskrit II
+        104937 # Altirisch
+        104925 # Gradierung und Komparation
+        105264 # Was andere Sprachen anders machen
+        104991 # Warum klingt Orkisch böse
+        105074 # Litauisch
+        103685 # Griechische Sprache und Übersetzung I
+        105455 # Elegia greca
+        105335 # Homerische Epen
+
+        # WS 2021
+        108122 # Griechisch
+        107986 # Altostslavisch
+        107792 # Elegie in Rom
+        107369 # Tusculanae Disputationes
+        108586 # Griechische Religion
+        107988 # Balkanindogermanisch
+        108312 # Altnordisch
+        107281 # NLP
+        108736 # Grammatiktheorie
+        109438 # Warum klingt Orkisch böse
+        108025 # Poetik der Nordgermanen
+        107280 # Statistik
+        108350 # Attische Redner
+        107461 # Argonautika
+        108283 # Digital Classicist
+        109211 # Altlitauisch
+        109185 # Etymologie
+      ];
+      download_submissions = true;
+      download_descriptions = true;
+      download_links_in_descriptions = false;
+      download_databases = false;
+      download_forums = false;
+      download_linked_files = false;
+      download_also_with_cookie = false;
+    };
+  };
+
+  fileSystems."/export/moodle" = {
+    device = config.services.moodle-dl.directory;
+    options = [ "bind" ];
+  };
+
+  networking.firewall.allowedTCPPorts = [ 2049 ];
+
+  services.nginx.virtualHosts."moodle.kmein.r" =
+  let
+    identity = lib.strings.fileContents <secrets/eduroam/identity>;
+    password = lib.strings.fileContents <secrets/eduroam/password>;
+  in {
+    basicAuth."${identity}" = password;
+    locations."/" = {
+      root = config.services.moodle-dl.directory;
+      extraConfig = ''
+        autoindex on;
+        charset UTF-8;
+      '';
+    };
+  };
+
+  services.nfs.server = {
+    enable = true;
+    exports = let machines = with (import <niveum/lib>).retiolumAddresses; [kabsa manakish]; in ''
+      /export        ${lib.concatMapStringsSep " " (machine: "${machine.ipv4}(fsid=0)") machines}
+      /export/moodle ${lib.concatMapStringsSep " " (machine: "${machine.ipv4}(insecure,rw)") machines}
+    '';
+  };
+}
--- a/systems/zaatar/pulseaudio.nix
+++ b/systems/zaatar/pulseaudio.nix
@@ -0,0 +1,14 @@
+{
+  sound.enable = true;
+
+  hardware.pulseaudio = {
+    enable = true;
+    systemWide = true;
+    tcp = {
+      enable = true;
+      anonymousClients.allowedIpRanges = [ "127.0.0.1" "10.243.2.0/24" "192.168.0.0/16" ];
+    };
+    zeroconf.publish.enable = true;
+  };
+  networking.firewall.allowedTCPPorts = [ 4713 ];
+}
--- a/systems/zaatar/tuna.nix
+++ b/systems/zaatar/tuna.nix
@@ -0,0 +1,114 @@
+{ config, pkgs, lib, ... }:
+let
+  firewall = (import <niveum/lib>).firewall lib;
+
+  streams = import <niveum/lib/streams.nix> {
+    di-fm-key = lib.strings.fileContents <secrets/di.fm/key>;
+  };
+  multi-room-audio-port = 8000;
+  password = lib.strings.fileContents <system-secrets/mpd-web.key>;
+in
+{
+  imports = [
+    <niveum/modules/tuna.nix>
+  ];
+
+  services.syncthing = let mpd-directory = config.services.mpd.dataDir; in {
+    enable = true;
+    user = config.services.mpd.user; # config.users.extraUsers.moodle.name;
+    openDefaultPorts = true;
+    configDir = "${mpd-directory}/.config/syncthing";
+    dataDir = "${mpd-directory}/.config/syncthing";
+    cert = toString <system-secrets/syncthing/cert.pem>;
+    key = toString <system-secrets/syncthing/key.pem>;
+    devices = {
+      inherit ((import <niveum/lib>).syncthing.devices) kabsa manakish heym;
+    };
+    folders.${config.services.mpd.musicDirectory} = {
+      devices = [ "heym" "kabsa" "manakish" ];
+      id = "music";
+      type = "receiveonly";
+    };
+  };
+
+  services.mpd = {
+    enable = true;
+    network.listenAddress = "0.0.0.0";
+    extraConfig = ''
+      log_level "default"
+      auto_update "yes"
+
+      audio_output {
+        type "pulse"
+        name "zaatar single room audio system"
+      }
+    '';
+  };
+
+  environment.systemPackages = [ pkgs.mpc_cli ];
+
+  networking.firewall =
+  let
+    dport = config.services.mpd.network.port;
+    protocol = "tcp";
+    rules = [
+      (firewall.accept { inherit dport protocol; source = "192.168.0.0/16"; })
+      (firewall.accept { inherit dport protocol; source = "127.0.0.0/8"; })
+    ];
+  in {
+    allowedTCPPorts = [ 80 ];
+    extraCommands = firewall.addRules rules;
+    extraStopCommands = firewall.removeRules rules;
+  };
+
+  system.activationScripts.mpd-playlists =
+  let playlistFile = pkgs.writeText "radio.m3u" (lib.concatMapStringsSep "\n" (lib.getAttr "stream") streams);
+  in ''
+    rm -rf /var/lib/mpd/playlists
+    install -d /var/lib/mpd/playlists
+    ln -sfn "${toString playlistFile}" "/var/lib/mpd/playlists/radio.m3u"
+  '';
+
+  services.tuna = {
+    enable = true;
+    # stationsFile = "/etc/tuna/stations.json";
+    stations = lib.lists.imap0 (id: {desc ? "", logo ? "https://picsum.photos/seed/${builtins.hashString "md5" stream}/300", stream, station}: { inherit id desc logo stream station; }) streams;
+    webPort = 8080;
+  };
+
+  systemd.services.tuna-stations =
+  let
+    stations = lib.lists.imap0 (id: {desc ? "", logo ? "https://picsum.photos/seed/${builtins.hashString "md5" stream}/300", stream, station}: { inherit id desc logo stream station; }) streams;
+    stationsJson = (pkgs.formats.json {}).generate "stations.json" stations;
+  in {
+    wantedBy = [ "tuna.service" ];
+    startAt = "hourly";
+    script = ''
+      mkdir -p /etc/tuna
+      antenne_asb_url=$(
+        ${pkgs.curl}/bin/curl -sS 'https://www.caster.fm/widgets/em_player.php?jsinit=true&uid=529295&t=blue&c=' \
+          | grep streamUrl \
+          | sed ${lib.escapeShellArg "s/^.*'\\([^']*\\)'.*/\\1/"}
+      )
+      ${pkgs.jq}/bin/jq "map(if .station == \"Antenne ASB\" then .stream |= \"$antenne_asb_url\" else . end)" < ${stationsJson} > /etc/tuna/stations.json
+    '';
+  };
+
+
+  /*
+  services.nginx = {
+    enable = true;
+    recommendedGzipSettings = true;
+    recommendedOptimisation = true;
+    recommendedProxySettings = true;
+    recommendedTlsSettings = true;
+    virtualHosts."radio.kmein.r" = {
+      basicAuth.dj = password;
+      locations."/" = {
+        proxyPass = "http://127.0.0.1:${toString config.services.tuna.webPort}";
+        proxyWebsockets = true;
+      };
+    };
+  };
+  */
+}