add examples and test VM

2026-02-20 18:14:49 +01:00
parent df73fc25e6
commit 8868eb8736
12 changed files with 325 additions and 416 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -0,0 +1 @@
+*.qcow2
--- a/19
+++ b/19
@@ -0,0 +1,19 @@
+Copyright © 2026 Kierán Meinhardt
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the “Software”), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
--- a/README.md
+++ b/README.md
@@ -1,9 +1,8 @@
 # Panoptikon – Watch the world from NixOS
+A NixOS module for monitoring website content and command output changes.

 ![](./panoptikon.jpg)

-A NixOS module for monitoring website content and command output changes.
-
 ## Overview

 Panoptikon is a generic command output and website watcher that periodically runs scripts and reports changes. It's designed to be flexible and can monitor anything from API endpoints to system metrics.
@@ -25,11 +24,6 @@ Add Panoptikon to your NixOS configuration:
 { config, pkgs, ... }:

 {
-  imports = [ 
-    # Add the Panoptikon module
-    (pkgs.callPackage (builtins.fetchTarball https://github.com/kfm/panoptikon-library/archive/main.tar.gz) { }).nixosModules.default
-  ];
-
  # Enable Panoptikon service
  services.panoptikon.enable = true;

@@ -51,21 +45,21 @@ Add Panoptikon to your NixOS configuration:
  services.panoptikon.watchers = {
    # Monitor GitHub metadata
    github-meta = {
-      script = pkgs.writers.writeDash "github-meta" '''
+      script = pkgs.writers.writeDash "github-meta" ''
        ${pkgs.curl}/bin/curl -sSL https://api.github.com/meta | ${pkgs.jq}/bin/jq
-      ''';
+      '';
      frequency = "*:0/5";  # Every 5 minutes
      reporters = [
        # Report changes to Telegram
-        (pkgs.writers.writeDash "telegram-reporter" '''
+        (pkgs.writers.writeDash "telegram-reporter" ''
          ${pkgs.curl}/bin/curl -X POST https://api.telegram.org/bot''${TOKEN}/sendMessage \
            -d chat_id=123456 \
            -d text="$(cat)"
-        ''')
+        '')
        # Also show desktop notifications
-        (pkgs.writers.writeDash "notify" '''
+        (pkgs.writers.writeDash "notify" ''
          ${pkgs.libnotify}/bin/notify-send "$PANOPTIKON_WATCHER has changed."
-        ''')
+        '')
      ];
    };

@@ -85,92 +79,24 @@ Add Panoptikon to your NixOS configuration:

    # Monitor a local command
    disk-space = {
-      script = pkgs.writers.writeDash "disk-space" '''
-        df -h / | tail -1 | awk '{print $5 " used"}'
-      ''';
+      script = pkgs.writers.writeDash "disk-space" ''
+        df -h / | tail -1 | awk '{print $5 " used
+        }'';
      frequency = "*:0/30";  # Every 30 minutes
      reporters = [
        # Log to systemd journal
-        (pkgs.writers.writeDash "journal-log" '''
+        (pkgs.writers.writeDash "journal-log" ''
          journalctl -t panoptikon-disk-space --since "1 hour ago" | tail -5
-        ''')
+        '')
      ];
    };
  };
 }
 ```

-### Advanced Configuration
-
-#### Using Secrets
-
-```nix
-{
-  services.panoptikon.watchers = {
-    private-api = {
-      script = pkgs.writers.writeDash "private-api" '''
-        ${pkgs.curl}/bin/curl -sSL \
-          -H "Authorization: Bearer $API_TOKEN" \
-          https://api.example.com/data
-      ''';
-      frequency = "hourly";
-      loadCredential = [ "API_TOKEN" ];
-      reporters = [
-        (pkgs.writers.writeDash "secure-reporter" '''
-          ${pkgs.curl}/bin/curl -X POST https://secure.example.com/report \
-            -d "$(cat)"
-        ''')
-      ];
-    };
-  };
-}
-```
-
-#### JSON API Monitoring
-
-```nix
-{
-  services.panoptikon.watchers = {
-    crypto-prices = {
-      script = pkgs.panoptikon.urlJSON {
-        jqScript = ".[0] | { name: .name, price: .quote.USD.price }";
-      } "https://api.coingecko.com/api/v3/coins/markets?vs_currency=usd&ids=bitcoin";
-      frequency = "*:0/15";  # Every 15 minutes
-      reporters = [
-        (pkgs.writers.writeDash "price-alert" '''
-          price=$(echo "$(cat)" | ${pkgs.jq}/bin/jq -r '.price')
-          if (( $(echo "$price > 60000" | bc -l) )); then
-            ${pkgs.libnotify}/bin/notify-send "Bitcoin price: $$price"
-          fi
-        ''')
-      ];
-    };
-  };
-}
-```
-
-## Available Helper Scripts
-
-Panoptikon provides several helper scripts in the overlay:
-
-### URL Monitoring
-
- `pkgs.panoptikon.url`: Fetch and convert HTML to text
- `pkgs.panoptikon.urlSelector`: Extract specific HTML elements using CSS selectors
- `pkgs.panoptikon.urlJSON`: Fetch JSON and apply jq transformations
-
-### Reporters
-
- `pkgs.panoptikon.kpaste-irc`: Report changes to IRC via kpaste
-  - `target`: IRC target (channel or user)
-  - `server`: IRC server (default: irc.r)
-  - `messagePrefix`: Prefix for messages (default: "change detected: ")
-  - `nick`: Nick to use (default: watcher name + "-watcher")
-  - `retiolumLink`: Use retiolum link (default: false)
-
 ## Service Management

-### Systemd Integration
+### systemd Integration

 Each watcher gets its own systemd service and timer:

@@ -190,11 +116,10 @@ systemctl start panoptikon-github-meta

 ### Timer Configuration

-Timers use systemd.timer syntax. Common examples:
+Timers use systemd timer syntax. Common examples:

 - `*:0/5` - Every 5 minutes
 - `daily` - Once per day
- `Mon..Fri 9:00-17:00` - Weekdays during business hours
 - `*:0/15` - Every 15 minutes
 - `weekly` - Once per week

@@ -204,135 +129,13 @@ See [systemd.time(7)](https://www.freedesktop.org/software/systemd/man/systemd.t

 - Watchers run as the `panoptikon` system user
 - Scripts are executed in `/var/lib/panoptikon`
- Use `loadCredential` to securely pass secrets
+- Use `LoadCredential=` to securely pass secrets
 - Scripts should be written defensively (use `set -euo pipefail`)

 ## Troubleshooting

-### Common Issues
-
-1. **No output**: Check if the script runs correctly manually
-2. **Permission denied**: Ensure the panoptikon user can access required resources
-3. **Network issues**: Add `network-online.target` as a dependency
-4. **Rate limiting**: Add randomized delays using `RandomizedDelaySec`
-
-### Debug Mode
-
-Enable debug logging:
-
-```nix
-services.panoptikon.watchers.github-meta = {
-  # ...
-  script = ''
-    set -x  # Enable debug output
-    ${pkgs.curl}/bin/curl -sSL https://api.github.com/meta | ${pkgs.jq}/bin/jq
-  '';
-};
-```
-
 ## Examples

-### Monitor Cryptocurrency Prices
+See the [examples directory](./examples/) for complete configurations.

-```nix
-{
-  services.panoptikon.enable = true;
-  
-  services.panoptikon.watchers = {
-    bitcoin-price = {
-      script = pkgs.panoptikon.urlJSON {
-        jqScript = ".[0] | { name: .name, price: .quote.USD.price, change: .quote.USD.percent_change_24h }";
-      } "https://api.coingecko.com/api/v3/coins/markets?vs_currency=usd&ids=bitcoin";
-      frequency = "*:0/30";
-      reporters = [
-        (pkgs.writers.writeDash "btc-alert" '''
-          price=$(echo "$(cat)" | ${pkgs.jq}/bin/jq -r '.price')
-          change=$(echo "$(cat)" | ${pkgs.jq}/bin/jq -r '.change')
-          
-          ${pkgs.libnotify}/bin/notify-send \
-            "Bitcoin: $$price ($$change%)"
-        ''')
-      ];
-    };
-  };
-}
-```
-
-### Monitor System Metrics
-
-```nix
-{
-  services.panoptikon.enable = true;
-  
-  services.panoptikon.watchers = {
-    load-average = {
-      script = pkgs.writers.writeDash "load-average" '''
-        uptime | awk -F'load average:' '{print $2}' | awk '{print $1 " (1m), " $2 " (5m), " $3 " (15m)"}'
-      ''';
-      frequency = "*:0/2";  # Every 2 minutes
-      reporters = [
-        (pkgs.writers.writeDash "load-alert" '''
-          load=$(echo "$(cat)" | awk '{print $1}' | tr -d ',')
-          if (( $(echo "$load > 2.0" | bc -l) )); then
-            ${pkgs.libnotify}/bin/notify-send "High load: $$load"
-          fi
-        ''')
-      ];
-    };
-  };
-}
-```
-
-### Monitor NixOS Updates
-
-```nix
-{
-  services.panoptikon.enable = true;
-  
-  services.panoptikon.watchers = {
-    nixos-updates = {
-      script = pkgs.panoptikon.url "https://nixos.org/blog/";
-      frequency = "daily";
-      reporters = [
-        (pkgs.writers.writeDash "update-notify" '''
-          ${pkgs.libnotify}/bin/notify-send "NixOS blog updated: $(cat | head -5)"
-        ''')
-      ];
-    };
-  };
-}
-```
-
-## Development
-
-### Adding Custom Reporters
-
-Create a new reporter in the overlay:
-
-```nix
-final: prev: {
-  panoptikon = prev.panoptikon // {
-    my-custom-reporter = {
-      endpoint,
-      authToken ? "",
-      ...
-    }:
-      prev.writers.writeDash "my-reporter" ''
-        ${prev.curl}/bin/curl -X POST ''${endpoint} \
-          -H "Authorization: Bearer ''${authToken} \
-          -d "$(cat)"
-      '';
-  };
-};
-```
-
-### Contributing
-
-1. Fork the repository
-2. Create a feature branch
-3. Add tests if applicable
-4. Submit a pull request
-
-## License
-
-MIT License
+Run `nix run .#panoptikon-vm` to start a VM with Panoptikon and example watchers pre-configured.
--- a/examples/advanced.nix
+++ b/examples/advanced.nix
@@ -1,119 +0,0 @@
-# Advanced Panoptikon configuration with secrets and custom reporters
-
-{
-  # Load secrets from agenix
-  secrets = import ../../secrets { };
-
-  services.panoptikon.enable = true;
-  
-  services.panoptikon.watchers = {
-    # Monitor a private API with authentication
-    private-api = {
-      script = pkgs.writers.writeDash "private-api" '''
-        set -euo pipefail
-        ${pkgs.curl}/bin/curl -sSL \
-          -H "Authorization: Bearer $API_TOKEN" \
-          -H "Content-Type: application/json" \
-          https://api.example.com/data
-      ''';
-      frequency = "hourly";
-      loadCredential = [ "API_TOKEN" ];
-      reporters = [
-        # Custom reporter that sends to a webhook
-        (pkgs.writers.writeDash "webhook-reporter" '''
-          ${pkgs.curl}/bin/curl -X POST \
-            -H "Content-Type: application/json" \
-            -d "{\"watcher\": \"$PANOPTIKON_WATCHER\", \"changes\": $(cat)}" \
-            https://hooks.example.com/panoptikon
-        ''')
-        # Also log to systemd journal
-        (pkgs.writers.writeDash "journal-log" '''
-          journalctl -t panoptikon-private-api --since "1 hour ago" | tail -5
-        ''')
-      ];
-    };
-
-    # Monitor cryptocurrency prices with alerts
-    crypto-monitor = {
-      script = pkgs.panoptikon.urlJSON {
-        jqScript = ".[0] | {
-          name: .name,
-          price: .quote.USD.price,
-          change24h: .quote.USD.percent_change_24h,
-          marketCap: .quote.USD.market_cap
-        }";
-      } "https://api.coingecko.com/api/v3/coins/markets?vs_currency=usd&ids=bitcoin";
-      frequency = "*:0/15";
-      reporters = [
-        (pkgs.writers.writeDash "btc-alert" '''
-          price=$(echo "$(cat)" | ${pkgs.jq}/bin/jq -r '.price')
-          change=$(echo "$(cat)" | ${pkgs.jq}/bin/jq -r '.change24h')
-          
-          # Alert if price > $60,000 or change > 5%
-          if (( $(echo "$price > 60000" | bc -l) )) || (( $(echo "$change > 5" | bc -l) )); then
-            ${pkgs.libnotify}/bin/notify-send \
-              "BTC Alert: $$price ($$change% change)"
-          fi
-        ''')
-        # Log to file
-        (pkgs.writers.writeDash "price-logger" '''
-          echo "$(date): $(cat)" >> /var/log/panoptikon/btc-prices.log
-        ''')
-      ];
-    };
-
-    # Monitor system load with thresholds
-    system-health = {
-      script = pkgs.writers.writeDash "system-health" '''
-        set -euo pipefail
-        load=$(uptime | awk -F'load average:' '{print $2}' | awk '{print $1}' | tr -d ',')
-        mem=$(free -m | awk 'NR==2{printf "%.1f%%", $3*100/$2 }')
-        disk=$(df / | awk 'NR==2{printf "%.1f%%", $5}')
-        
-        echo "load: $$load, mem: $$mem, disk: $$disk"
-      ''';
-      frequency = "*:0/5";
-      reporters = [
-        (pkgs.writers.writeDash "health-alert" '''
-          load=$(echo "$(cat)" | awk -F',' '{print $1}' | awk '{print $2}')
-          mem=$(echo "$(cat)" | awk -F',' '{print $2}' | awk '{print $2}')
-          disk=$(echo "$(cat)" | awk -F',' '{print $3}' | awk '{print $2}')
-          
-          # Alert if load > 2.0, mem > 80%, or disk > 90%
-          if (( $(echo "$load > 2.0" | bc -l) )) || (( $(echo "${mem%%%} > 80" | bc -l) )) || (( $(echo "${disk%%%} > 90" | bc -l) )); then
-            ${pkgs.libnotify}/bin/notify-send \
-              "System Alert: Load=$$load, Mem=$$mem, Disk=$$disk"
-          fi
-        ''')
-      ];
-    };
-  };
-
-  # Add monitoring user
-  users.extraUsers.panoptikon = {
-    isSystemUser = true;
-    createHome = true;
-    home = "/var/lib/panoptikon";
-    group = "panoptikon";
-    description = "Panoptikon monitoring service";
-    openssh.authorizedKeys.keys = [
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK..."  # Monitoring access key
-    ];
-  };
-
-  # Configure log rotation
-  services.logrotate = {
-    enable = true;
-    config = {
-      rotate = 14;
-      compress = true;
-      delaycompress = true;
-      missingok = true;
-      notifempty = true;
-      create = "644 panoptikon panoptikon";
-    };
-    files = [
-      "/var/log/panoptikon/*.log"
-    ];
-  };
-}
--- a/examples/bitcoin.nix
+++ b/examples/bitcoin.nix
@@ -0,0 +1,18 @@
+{ pkgs, ... }:
+{
+  services.panoptikon.enable = true;
+  services.panoptikon.watchers = {
+    bitcoin-price = {
+      script = pkgs.panoptikonWatchers.json {
+        jqScript = ".[]|{name: .name, price: .current_price, change: .price_change_24h}";
+      } "https://api.coingecko.com/api/v3/coins/markets?vs_currency=usd&ids=bitcoin";
+      frequency = "*:0/30";
+      reporters = [
+        (pkgs.panoptikonReporters.telegram {
+          chatId = "123";
+          tokenPath = pkgs.writeText "my-telegram-token.txt" "123:abc";
+        })
+      ];
+    };
+  };
+}
--- a/examples/nixos.nix
+++ b/examples/nixos.nix
@@ -0,0 +1,15 @@
+{ pkgs, ... }:
+{
+  services.panoptikon.enable = true;
+
+  services.panoptikon.watchers = {
+    nixos-updates = {
+      script = pkgs.panoptikonWatchers.html "https://nixos.org/blog/";
+      frequency = "daily";
+      reporters = [
+        (pkgs.panoptikonReporters.mail { recipient = "admin@example.org"; })
+      ];
+    };
+  };
+}
+
--- a/examples/simple.nix
+++ b/examples/simple.nix
@@ -1,30 +1,45 @@
-# Simple Panoptikon configuration
-
+{ pkgs, ... }:
 {
  services.panoptikon.enable = true;
-  
+
  services.panoptikon.watchers = {
-    # Monitor GitHub metadata every 5 minutes
    github-meta = {
-      script = pkgs.writers.writeDash "github-meta" '''
-        ${pkgs.curl}/bin/curl -sSL https://api.github.com/meta | ${pkgs.jq}/bin/jq
-      ''';
+      script = pkgs.panoptikonWatchers.json { } "https://api.github.com/meta";
      frequency = "*:0/5";
      reporters = [
-        (pkgs.writers.writeDash "notify" '''
-          ${pkgs.libnotify}/bin/notify-send "$PANOPTIKON_WATCHER has changed."
-        ''')
+        (pkgs.panoptikonReporters.wall { })
+        (pkgs.panoptikonReporters.irc {
+          target = "kmein";
+        })
+      ];
+    };
+
+    cock-canary = {
+      script = pkgs.panoptikonWatchers.plain "https://cock.li/canary.asc.txt";
+      frequency = "daily";
+      reporters = [
+        (pkgs.panoptikonReporters.irc {
+          target = "kmein";
+        })
+      ];
+    };
+
+    "4d2-canary" = {
+      script = pkgs.panoptikonWatchers.plain "https://4d2.org/canary.txt";
+      frequency = "daily";
+      reporters = [
+        (pkgs.panoptikonReporters.irc {
+          target = "kmein";
+        })
      ];
    };

    # Monitor a website for news
    nixos-news = {
-      script = pkgs.panoptikon.urlSelector "#news h2" "https://nixos.org/blog/";
+      script = pkgs.panoptikonWatchers.htmlSelector "article h2" "https://nixos.org/blog/";
      frequency = "daily";
      reporters = [
-        (pkgs.writers.writeDash "news-alert" '''
-          ${pkgs.libnotify}/bin/notify-send "New NixOS blog post: $(cat | head -1)"
-        ''')
+        (pkgs.panoptikonReporters.wall { })
      ];
    };
  };
--- a/examples/system.nix
+++ b/examples/system.nix
@@ -0,0 +1,16 @@
+{ pkgs, ... }:
+{
+  services.panoptikon.enable = true;
+
+  services.panoptikon.watchers = {
+    boot-space = {
+      script = pkgs.writers.writeDash "load-average" ''
+        ${pkgs.coreutils}/bin/df -h /boot
+      '';
+      frequency = "*:0/2";
+      reporters = [
+        (pkgs.panoptikonReporters.wall { })
+      ];
+    };
+  };
+}
--- a/flake.nix
+++ b/flake.nix
@@ -1,13 +1,50 @@
 {
-  description = "Panoptikon - Website and command output monitoring";
+  description = "Panoptikon – Watch the world from NixOS";

  inputs = {
    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
  };

-  outputs = inputs: {
-    nixosModules.default = ./module.nix;
-    overlays.default = ./overlay.nix;
-  };
-}
+  outputs =
+    inputs:
+    let
+      eachSupportedSystem = inputs.nixpkgs.lib.genAttrs inputs.nixpkgs.lib.systems.flakeExposed;
+      inherit (inputs.nixpkgs) lib;
+    in
+    {
+      nixosModules.default = import nix/module.nix;
+      overlays.default = import nix/overlay.nix;

+      apps = eachSupportedSystem (
+        system:
+        let
+          nixosSystem = lib.nixosSystem {
+            system = "x86_64-linux";
+            modules = [
+              inputs.self.nixosModules.default
+              {
+                nixpkgs.overlays = [ inputs.self.overlay ];
+              }
+              {
+                virtualisation.vmVariant = {
+                  virtualisation.graphics = false;
+                };
+                services.getty.autologinUser = "root";
+                system.stateVersion = lib.trivial.release;
+              }
+              ./examples/simple.nix
+              ./examples/bitcoin.nix
+              ./examples/nixos.nix
+              ./examples/system.nix
+            ];
+          };
+        in
+        {
+          panoptikon-vm = {
+            type = "app";
+            program = lib.getExe nixosSystem.config.system.build.vm;
+          };
+        }
+      );
+    };
+}
--- a/nix/module.nix
+++ b/nix/module.nix
@@ -65,12 +65,13 @@
  config =
    let
      cfg = config.services.panoptikon;
+      stateDir = "/var/lib/panoptikon";
    in
    lib.mkIf cfg.enable {
      users.extraUsers.panoptikon = {
        isSystemUser = true;
        createHome = true;
-        home = "/var/lib/panoptikon";
+        home = stateDir;
        group = "panoptikon";
      };

@@ -85,37 +86,62 @@

      systemd.services = lib.attrsets.mapAttrs' (
        watcherName: watcherOptions:
+        let
+          # Absolute paths for the state files
+          currentFile = "${stateDir}/${watcherName}";
+          oldFile = "${stateDir}/${watcherName}.old";
+        in
        lib.nameValuePair "panoptikon-${watcherName}" {
          enable = true;
          startAt = watcherOptions.frequency;
+
          serviceConfig = {
            Type = "oneshot";
            User = "panoptikon";
            Group = "panoptikon";
-            WorkingDirectory = "/var/lib/panoptikon";
-            RestartSec = toString (60 * 60);
+
+            # ISOLATION: Start in a fresh, empty directory for every run
+            RuntimeDirectory = "panoptikon/${watcherName}";
+            WorkingDirectory = "/run/panoptikon/${watcherName}";
+
+            # PERSISTENCE: Only allow write access to the specific state directory
+            ReadWritePaths = [ stateDir ];
+
+            # HARDENING: Prevent the process from seeing most of the system
+            ProtectSystem = "strict";
+            ProtectHome = true;
+            PrivateTmp = true;
+            NoNewPrivileges = true;
+
            Restart = "on-failure";
            LoadCredential = watcherOptions.loadCredential;
          };
-          unitConfig = {
-            StartLimitIntervalSec = "300";
-            StartLimitBurst = "5";
-          };
+
          environment.PANOPTIKON_WATCHER = watcherName;
-          wants = [ "network-online.target" ];
+
          script = ''
-            set -fux
-            ${watcherOptions.script} > ${lib.escapeShellArg watcherName}
-            diff_output=$(${pkgs.diffutils}/bin/diff --new-file ${
-              lib.escapeShellArg (watcherName + ".old")
-            } ${lib.escapeShellArg watcherName} || :)
-            if [ -n "$diff_output" ]
-            then
+            set -efu # Removed -x to keep tokens out of logs if they leak via env
+
+            # 1. Run watcher and save to the persistent state dir
+            ${watcherOptions.script} > "${currentFile}"
+
+            # 2. Ensure .old exists so diff doesn't crash on first run
+            [ -f "${oldFile}" ] || touch "${oldFile}"
+
+            # 3. Compare
+            diff_output=$(${pkgs.diffutils}/bin/diff --new-file "${oldFile}" "${currentFile}" || :)
+
+            if [ -n "$diff_output" ]; then
+              # 4. Run reporters.
+              # They are running inside /run/panoptikon/${watcherName} (empty).
+              # They can't see or delete other .old files via relative paths.
              ${lib.strings.concatMapStringsSep "\n" (
                reporter: ''echo "$diff_output" | ${reporter} || :''
              ) watcherOptions.reporters}
            fi
-            mv ${lib.escapeShellArg watcherName} ${lib.escapeShellArg (watcherName + ".old")}
+
+            # 5. Rotate state
+            mv "${currentFile}" "${oldFile}"
          '';
        }
      ) cfg.watchers;
--- a/nix/overlay.nix
+++ b/nix/overlay.nix
@@ -0,0 +1,125 @@
+final: prev:
+let
+  lib = prev.lib;
+in
+{
+  panoptikonReporters = (prev.panoptikonWatchers or { }) // {
+    kpaste-irc =
+      {
+        target,
+        retiolumLink ? false,
+        server ? "irc.r",
+        messagePrefix ? "change detected: ",
+        nick ? ''"$PANOPTIKON_WATCHER"-watcher'',
+      }:
+      prev.writers.writeDash "kpaste-irc-reporter" ''
+        KPASTE_CONTENT_TYPE=text/plain ${prev.kpaste}/bin/kpaste \
+          | ${prev.gnused}/bin/sed -n "${if retiolumLink then "2" else "3"}s/^/${messagePrefix}/p" \
+          | ${prev.nur.repos.mic92.ircsink}/bin/ircsink \
+            --nick ${nick} \
+            --server ${server} \
+            --target ${target}
+      '';
+    irc =
+      {
+        target, # e.g., "#mychannel"
+        server ? "irc.libera.chat",
+        port ? "6667",
+        nick ? "panoptikon-bot",
+        messagePrefix ? "change detected: ",
+      }:
+      prev.writers.writeDash "irc-reporter" ''
+        MESSAGE=$(cat | tr -d '\n\r')
+        # Use netcat to send raw IRC commands
+        (
+          echo "NICK ${nick}"
+          echo "USER ${nick} 8 * :Panoptikon Watcher"
+          sleep 2 # Give the server a moment to process login
+          echo "JOIN ${target}"
+          echo "PRIVMSG ${target} :${messagePrefix} $MESSAGE"
+          echo "QUIT"
+        ) | ${prev.netcat}/bin/nc -w 10 ${server} ${port}
+      '';
+    telegram =
+      {
+        tokenPath, # Bot API Token
+        chatId,
+        messagePrefix ? "Change detected in $PANOPTIKON_WATCHER: ",
+      }:
+      prev.writers.writeDash "telegram-reporter" ''
+        if [ ! -f "${tokenPath}" ]; then
+          echo "Error: Token file ${tokenPath} not found" >&2
+          exit 1
+        fi
+        TOKEN=$(cat "${tokenPath}")
+        MESSAGE=$(cat)
+        ${prev.curl}/bin/curl -s -X POST "https://api.telegram.org/bot''${TOKEN}/sendMessage" \
+          -d chat_id="${chatId}" \
+          -d text="${messagePrefix} $MESSAGE"
+      '';
+    matrix =
+      {
+        homeserver, # e.g., "https://matrix.org"
+        roomId, # e.g., "!roomid:matrix.org"
+        tokenPath,
+        messagePrefix ? "Change detected in $PANOPTIKON_WATCHER: ",
+      }:
+      prev.writers.writeDash "matrix-reporter" ''
+        if [ ! -f "${tokenPath}" ]; then
+          echo "Error: Token file ${tokenPath} not found" >&2
+          exit 1
+        fi
+        TOKEN=$(cat "${tokenPath}")
+        MESSAGE=$(cat)
+        TXN_ID=$(date +%s%N)
+        ${prev.curl}/bin/curl -s -X PUT "${homeserver}/_matrix/client/r0/rooms/${lib.escapeShellArg roomId}/send/m.room.message/''${TXN_ID}?access_token=''${TOKEN}" \
+          -H "Content-Type: application/json" \
+          -d "{\"msgtype\": \"m.text\", \"body\": \"${messagePrefix} $MESSAGE\"}"
+      '';
+    wall =
+      {
+        messagePrefix ? "PANOPTIKON ALERT ($PANOPTIKON_WATCHER):",
+      }:
+      prev.writers.writeDash "wall-reporter" ''
+        MESSAGE=$(cat)
+        echo -e "${messagePrefix}\n$MESSAGE" | ${prev.util-linux}/bin/wall
+      '';
+    mail =
+      {
+        recipient, # e.g. "admin@example.com"
+        subjectPrefix ? "[Panoptikon] Change detected:",
+      }:
+      prev.writers.writeDash "mail-reporter" ''
+        MESSAGE=$(cat)
+        echo "$MESSAGE" | ${prev.mailutils}/bin/mail -s "${subjectPrefix} $PANOPTIKON_WATCHER" "${recipient}"
+      '';
+  };
+  panoptikonWatchers = (prev.panoptikonWatchers or { }) // {
+    plain =
+      address:
+      prev.writers.writeDash "watch-url" ''
+        ${prev.curl}/bin/curl -sSL ${lib.escapeShellArg address}
+      '';
+    html =
+      address:
+      prev.writers.writeDash "watch-html" ''
+        ${prev.curl}/bin/curl -sSL ${lib.escapeShellArg address} \
+          | ${prev.python3Packages.html2text}/bin/html2text --decode-errors=ignore
+      '';
+    htmlSelector =
+      selector: address:
+      prev.writers.writeDash "watch-html-selector" ''
+        ${prev.curl}/bin/curl -sSL ${lib.escapeShellArg address} \
+          | ${prev.htmlq}/bin/htmlq ${lib.escapeShellArg selector} \
+          | ${prev.python3Packages.html2text}/bin/html2text
+      '';
+    json =
+      {
+        jqScript ? ".",
+      }:
+      address:
+      prev.writers.writeDash "watch-json" ''
+        ${prev.curl}/bin/curl -sSL ${lib.escapeShellArg address} | ${prev.jq}/bin/jq -f ${prev.writeText "script.jq" jqScript}
+      '';
+  };
+}
--- a/overlay.nix
+++ b/overlay.nix
@@ -1,47 +0,0 @@
-final: prev: {
-  panoptikon =
-    let
-      lib = prev.lib;
-    in
-    lib.recursiveUpdate prev.panoptikon {
-      url =
-        address:
-        prev.writers.writeDash "watch-url" ''
-          ${prev.curl}/bin/curl -sSL ${lib.escapeShellArg address} \
-            | ${prev.python3Packages.html2text}/bin/html2text --decode-errors=ignore
-        '';
-      urlSelector =
-        selector: address:
-        prev.writers.writeDash "watch-url-selector" ''
-          ${prev.curl}/bin/curl -sSL ${lib.escapeShellArg address} \
-            | ${prev.htmlq}/bin/htmlq ${lib.escapeShellArg selector} \
-            | ${prev.python3Packages.html2text}/bin/html2text
-        '';
-      urlJSON =
-        {
-          jqScript ? ".",
-        }:
-        address:
-        prev.writers.writeDash "watch-url-json" ''
-          ${prev.curl}/bin/curl -sSL ${lib.escapeShellArg address} | ${prev.jq}/bin/jq -f ${prev.writeText "script.jq" jqScript}
-        '';
-
-      # reporter scripts
-      kpaste-irc =
-        {
-          target,
-          retiolumLink ? false,
-          server ? "irc.r",
-          messagePrefix ? "change detected: ",
-          nick ? ''"$PANOPTIKON_WATCHER"-watcher'',
-        }:
-        prev.writers.writeDash "kpaste-irc-reporter" ''
-          KPASTE_CONTENT_TYPE=text/plain ${prev.kpaste}/bin/kpaste \
-            | ${prev.gnused}/bin/sed -n "${if retiolumLink then "2" else "3"}s/^/${messagePrefix}/p" \
-            | ${prev.nur.repos.mic92.ircsink}/bin/ircsink \
-              --nick ${nick} \
-              --server ${server} \
-              --target ${target}
-        '';
-    };
-}