initial commit

2026-02-20 17:22:14 +01:00
commit 86dc6ff2d9
7 changed files with 696 additions and 0 deletions
--- a/README.md
+++ b/README.md
@@ -0,0 +1,336 @@
+# Panoptikon - Website and Command Output Monitoring
+
+A NixOS module for monitoring website content and command output changes.
+
+## Overview
+
+Panoptikon is a generic command output and website watcher that periodically runs scripts and reports changes. It's designed to be flexible and can monitor anything from API endpoints to system metrics.
+
+## Features
+
+- **Flexible Watchers**: Monitor any command output or website content
+- **Custom Frequencies**: Run scripts at any interval using systemd.timer syntax
+- **Multiple Reporters**: Report changes to various destinations (IRC, Telegram, Prometheus, etc.)
+- **Secret Support**: Securely pass credentials to scripts without exposing them in the Nix store
+- **Stateful Tracking**: Automatically tracks previous output and reports only changes
+- **Modular Design**: Easy to extend with custom watchers and reporters
+
+## Installation
+
+Add Panoptikon to your NixOS configuration:
+
+```nix
+{ config, pkgs, ... }:
+
+{
+  imports = [ 
+    # Add the Panoptikon module
+    (pkgs.callPackage (builtins.fetchTarball https://github.com/kfm/panoptikon-library/archive/main.tar.gz) { }).nixosModules.default
+  ];
+
+  # Enable Panoptikon service
+  services.panoptikon.enable = true;
+
+  # Configure your watchers
+  services.panoptikon.watchers = {
+    # Your watcher configurations go here
+  };
+}
+```
+
+## Configuration
+
+### Basic Watcher Configuration
+
+```nix
+{
+  services.panoptikon.enable = true;
+  
+  services.panoptikon.watchers = {
+    # Monitor GitHub metadata
+    github-meta = {
+      script = pkgs.writers.writeDash "github-meta" '''
+        ${pkgs.curl}/bin/curl -sSL https://api.github.com/meta | ${pkgs.jq}/bin/jq
+      ''';
+      frequency = "*:0/5";  # Every 5 minutes
+      reporters = [
+        # Report changes to Telegram
+        (pkgs.writers.writeDash "telegram-reporter" '''
+          ${pkgs.curl}/bin/curl -X POST https://api.telegram.org/bot''${TOKEN}/sendMessage \
+            -d chat_id=123456 \
+            -d text="$(cat)"
+        ''')
+        # Also show desktop notifications
+        (pkgs.writers.writeDash "notify" '''
+          ${pkgs.libnotify}/bin/notify-send "$PANOPTIKON_WATCHER has changed."
+        ''')
+      ];
+    };
+
+    # Monitor a website for specific content
+    nixos-updates = {
+      script = pkgs.panoptikon.urlSelector "#news h2" "https://nixos.org/blog/";
+      frequency = "daily";
+      reporters = [
+        # Report to IRC
+        (pkgs.panoptikon.kpaste-irc {
+          target = "#nixos";
+          server = "irc.libera.chat";
+          messagePrefix = "New NixOS blog post: ";
+        })
+      ];
+    };
+
+    # Monitor a local command
+    disk-space = {
+      script = pkgs.writers.writeDash "disk-space" '''
+        df -h / | tail -1 | awk '{print $5 " used"}'
+      ''';
+      frequency = "*:0/30";  # Every 30 minutes
+      reporters = [
+        # Log to systemd journal
+        (pkgs.writers.writeDash "journal-log" '''
+          journalctl -t panoptikon-disk-space --since "1 hour ago" | tail -5
+        ''')
+      ];
+    };
+  };
+}
+```
+
+### Advanced Configuration
+
+#### Using Secrets
+
+```nix
+{
+  services.panoptikon.watchers = {
+    private-api = {
+      script = pkgs.writers.writeDash "private-api" '''
+        ${pkgs.curl}/bin/curl -sSL \
+          -H "Authorization: Bearer $API_TOKEN" \
+          https://api.example.com/data
+      ''';
+      frequency = "hourly";
+      loadCredential = [ "API_TOKEN" ];
+      reporters = [
+        (pkgs.writers.writeDash "secure-reporter" '''
+          ${pkgs.curl}/bin/curl -X POST https://secure.example.com/report \
+            -d "$(cat)"
+        ''')
+      ];
+    };
+  };
+}
+```
+
+#### JSON API Monitoring
+
+```nix
+{
+  services.panoptikon.watchers = {
+    crypto-prices = {
+      script = pkgs.panoptikon.urlJSON {
+        jqScript = ".[0] | { name: .name, price: .quote.USD.price }";
+      } "https://api.coingecko.com/api/v3/coins/markets?vs_currency=usd&ids=bitcoin";
+      frequency = "*:0/15";  # Every 15 minutes
+      reporters = [
+        (pkgs.writers.writeDash "price-alert" '''
+          price=$(echo "$(cat)" | ${pkgs.jq}/bin/jq -r '.price')
+          if (( $(echo "$price > 60000" | bc -l) )); then
+            ${pkgs.libnotify}/bin/notify-send "Bitcoin price: $$price"
+          fi
+        ''')
+      ];
+    };
+  };
+}
+```
+
+## Available Helper Scripts
+
+Panoptikon provides several helper scripts in the overlay:
+
+### URL Monitoring
+
+- `pkgs.panoptikon.url`: Fetch and convert HTML to text
+- `pkgs.panoptikon.urlSelector`: Extract specific HTML elements using CSS selectors
+- `pkgs.panoptikon.urlJSON`: Fetch JSON and apply jq transformations
+
+### Reporters
+
+- `pkgs.panoptikon.kpaste-irc`: Report changes to IRC via kpaste
+  - `target`: IRC target (channel or user)
+  - `server`: IRC server (default: irc.r)
+  - `messagePrefix`: Prefix for messages (default: "change detected: ")
+  - `nick`: Nick to use (default: watcher name + "-watcher")
+  - `retiolumLink`: Use retiolum link (default: false)
+
+## Service Management
+
+### Systemd Integration
+
+Each watcher gets its own systemd service and timer:
+
+```bash
+# List all Panoptikon services
+systemctl list-units "panoptikon-*"
+
+# Check a specific watcher
+systemctl status panoptikon-github-meta
+
+# View logs
+journalctl -u panoptikon-github-meta -f
+
+# Trigger a manual run
+systemctl start panoptikon-github-meta
+```
+
+### Timer Configuration
+
+Timers use systemd.timer syntax. Common examples:
+
+- `*:0/5` - Every 5 minutes
+- `daily` - Once per day
+- `Mon..Fri 9:00-17:00` - Weekdays during business hours
+- `*:0/15` - Every 15 minutes
+- `weekly` - Once per week
+
+See [systemd.time(7)](https://www.freedesktop.org/software/systemd/man/systemd.time.html) for full syntax.
+
+## Security Considerations
+
+- Watchers run as the `panoptikon` system user
+- Scripts are executed in `/var/lib/panoptikon`
+- Use `loadCredential` to securely pass secrets
+- Scripts should be written defensively (use `set -euo pipefail`)
+
+## Troubleshooting
+
+### Common Issues
+
+1. **No output**: Check if the script runs correctly manually
+2. **Permission denied**: Ensure the panoptikon user can access required resources
+3. **Network issues**: Add `network-online.target` as a dependency
+4. **Rate limiting**: Add randomized delays using `RandomizedDelaySec`
+
+### Debug Mode
+
+Enable debug logging:
+
+```nix
+services.panoptikon.watchers.github-meta = {
+  # ...
+  script = ''
+    set -x  # Enable debug output
+    ${pkgs.curl}/bin/curl -sSL https://api.github.com/meta | ${pkgs.jq}/bin/jq
+  '';
+};
+```
+
+## Examples
+
+### Monitor Cryptocurrency Prices
+
+```nix
+{
+  services.panoptikon.enable = true;
+  
+  services.panoptikon.watchers = {
+    bitcoin-price = {
+      script = pkgs.panoptikon.urlJSON {
+        jqScript = ".[0] | { name: .name, price: .quote.USD.price, change: .quote.USD.percent_change_24h }";
+      } "https://api.coingecko.com/api/v3/coins/markets?vs_currency=usd&ids=bitcoin";
+      frequency = "*:0/30";
+      reporters = [
+        (pkgs.writers.writeDash "btc-alert" '''
+          price=$(echo "$(cat)" | ${pkgs.jq}/bin/jq -r '.price')
+          change=$(echo "$(cat)" | ${pkgs.jq}/bin/jq -r '.change')
+          
+          ${pkgs.libnotify}/bin/notify-send \
+            "Bitcoin: $$price ($$change%)"
+        ''')
+      ];
+    };
+  };
+}
+```
+
+### Monitor System Metrics
+
+```nix
+{
+  services.panoptikon.enable = true;
+  
+  services.panoptikon.watchers = {
+    load-average = {
+      script = pkgs.writers.writeDash "load-average" '''
+        uptime | awk -F'load average:' '{print $2}' | awk '{print $1 " (1m), " $2 " (5m), " $3 " (15m)"}'
+      ''';
+      frequency = "*:0/2";  # Every 2 minutes
+      reporters = [
+        (pkgs.writers.writeDash "load-alert" '''
+          load=$(echo "$(cat)" | awk '{print $1}' | tr -d ',')
+          if (( $(echo "$load > 2.0" | bc -l) )); then
+            ${pkgs.libnotify}/bin/notify-send "High load: $$load"
+          fi
+        ''')
+      ];
+    };
+  };
+}
+```
+
+### Monitor NixOS Updates
+
+```nix
+{
+  services.panoptikon.enable = true;
+  
+  services.panoptikon.watchers = {
+    nixos-updates = {
+      script = pkgs.panoptikon.url "https://nixos.org/blog/";
+      frequency = "daily";
+      reporters = [
+        (pkgs.writers.writeDash "update-notify" '''
+          ${pkgs.libnotify}/bin/notify-send "NixOS blog updated: $(cat | head -5)"
+        ''')
+      ];
+    };
+  };
+}
+```
+
+## Development
+
+### Adding Custom Reporters
+
+Create a new reporter in the overlay:
+
+```nix
+final: prev: {
+  panoptikon = prev.panoptikon // {
+    my-custom-reporter = {
+      endpoint,
+      authToken ? "",
+      ...
+    }:
+      prev.writers.writeDash "my-reporter" ''
+        ${prev.curl}/bin/curl -X POST ''${endpoint} \
+          -H "Authorization: Bearer ''${authToken} \
+          -d "$(cat)"
+      '';
+  };
+};
+```
+
+### Contributing
+
+1. Fork the repository
+2. Create a feature branch
+3. Add tests if applicable
+4. Submit a pull request
+
+## License
+
+MIT License
--- a/examples/advanced.nix
+++ b/examples/advanced.nix
@@ -0,0 +1,119 @@
+# Advanced Panoptikon configuration with secrets and custom reporters
+
+{
+  # Load secrets from agenix
+  secrets = import ../../secrets { };
+
+  services.panoptikon.enable = true;
+  
+  services.panoptikon.watchers = {
+    # Monitor a private API with authentication
+    private-api = {
+      script = pkgs.writers.writeDash "private-api" '''
+        set -euo pipefail
+        ${pkgs.curl}/bin/curl -sSL \
+          -H "Authorization: Bearer $API_TOKEN" \
+          -H "Content-Type: application/json" \
+          https://api.example.com/data
+      ''';
+      frequency = "hourly";
+      loadCredential = [ "API_TOKEN" ];
+      reporters = [
+        # Custom reporter that sends to a webhook
+        (pkgs.writers.writeDash "webhook-reporter" '''
+          ${pkgs.curl}/bin/curl -X POST \
+            -H "Content-Type: application/json" \
+            -d "{\"watcher\": \"$PANOPTIKON_WATCHER\", \"changes\": $(cat)}" \
+            https://hooks.example.com/panoptikon
+        ''')
+        # Also log to systemd journal
+        (pkgs.writers.writeDash "journal-log" '''
+          journalctl -t panoptikon-private-api --since "1 hour ago" | tail -5
+        ''')
+      ];
+    };
+
+    # Monitor cryptocurrency prices with alerts
+    crypto-monitor = {
+      script = pkgs.panoptikon.urlJSON {
+        jqScript = ".[0] | {
+          name: .name,
+          price: .quote.USD.price,
+          change24h: .quote.USD.percent_change_24h,
+          marketCap: .quote.USD.market_cap
+        }";
+      } "https://api.coingecko.com/api/v3/coins/markets?vs_currency=usd&ids=bitcoin";
+      frequency = "*:0/15";
+      reporters = [
+        (pkgs.writers.writeDash "btc-alert" '''
+          price=$(echo "$(cat)" | ${pkgs.jq}/bin/jq -r '.price')
+          change=$(echo "$(cat)" | ${pkgs.jq}/bin/jq -r '.change24h')
+          
+          # Alert if price > $60,000 or change > 5%
+          if (( $(echo "$price > 60000" | bc -l) )) || (( $(echo "$change > 5" | bc -l) )); then
+            ${pkgs.libnotify}/bin/notify-send \
+              "BTC Alert: $$price ($$change% change)"
+          fi
+        ''')
+        # Log to file
+        (pkgs.writers.writeDash "price-logger" '''
+          echo "$(date): $(cat)" >> /var/log/panoptikon/btc-prices.log
+        ''')
+      ];
+    };
+
+    # Monitor system load with thresholds
+    system-health = {
+      script = pkgs.writers.writeDash "system-health" '''
+        set -euo pipefail
+        load=$(uptime | awk -F'load average:' '{print $2}' | awk '{print $1}' | tr -d ',')
+        mem=$(free -m | awk 'NR==2{printf "%.1f%%", $3*100/$2 }')
+        disk=$(df / | awk 'NR==2{printf "%.1f%%", $5}')
+        
+        echo "load: $$load, mem: $$mem, disk: $$disk"
+      ''';
+      frequency = "*:0/5";
+      reporters = [
+        (pkgs.writers.writeDash "health-alert" '''
+          load=$(echo "$(cat)" | awk -F',' '{print $1}' | awk '{print $2}')
+          mem=$(echo "$(cat)" | awk -F',' '{print $2}' | awk '{print $2}')
+          disk=$(echo "$(cat)" | awk -F',' '{print $3}' | awk '{print $2}')
+          
+          # Alert if load > 2.0, mem > 80%, or disk > 90%
+          if (( $(echo "$load > 2.0" | bc -l) )) || (( $(echo "${mem%%%} > 80" | bc -l) )) || (( $(echo "${disk%%%} > 90" | bc -l) )); then
+            ${pkgs.libnotify}/bin/notify-send \
+              "System Alert: Load=$$load, Mem=$$mem, Disk=$$disk"
+          fi
+        ''')
+      ];
+    };
+  };
+
+  # Add monitoring user
+  users.extraUsers.panoptikon = {
+    isSystemUser = true;
+    createHome = true;
+    home = "/var/lib/panoptikon";
+    group = "panoptikon";
+    description = "Panoptikon monitoring service";
+    openssh.authorizedKeys.keys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK..."  # Monitoring access key
+    ];
+  };
+
+  # Configure log rotation
+  services.logrotate = {
+    enable = true;
+    config = {
+      rotate = 14;
+      compress = true;
+      delaycompress = true;
+      missingok = true;
+      notifempty = true;
+      create = "644 panoptikon panoptikon";
+    };
+    files = [
+      "/var/log/panoptikon/*.log"
+    ];
+  };
+}
--- a/examples/simple.nix
+++ b/examples/simple.nix
@@ -0,0 +1,31 @@
+# Simple Panoptikon configuration
+
+{
+  services.panoptikon.enable = true;
+  
+  services.panoptikon.watchers = {
+    # Monitor GitHub metadata every 5 minutes
+    github-meta = {
+      script = pkgs.writers.writeDash "github-meta" '''
+        ${pkgs.curl}/bin/curl -sSL https://api.github.com/meta | ${pkgs.jq}/bin/jq
+      ''';
+      frequency = "*:0/5";
+      reporters = [
+        (pkgs.writers.writeDash "notify" '''
+          ${pkgs.libnotify}/bin/notify-send "$PANOPTIKON_WATCHER has changed."
+        ''')
+      ];
+    };
+
+    # Monitor a website for news
+    nixos-news = {
+      script = pkgs.panoptikon.urlSelector "#news h2" "https://nixos.org/blog/";
+      frequency = "daily";
+      reporters = [
+        (pkgs.writers.writeDash "news-alert" '''
+          ${pkgs.libnotify}/bin/notify-send "New NixOS blog post: $(cat | head -1)"
+        ''')
+      ];
+    };
+  };
+}
--- a/flake.lock
+++ b/flake.lock
@@ -0,0 +1,27 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1771369470,
+        "narHash": "sha256-0NBlEBKkN3lufyvFegY4TYv5mCNHbi5OmBDrzihbBMQ=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "0182a361324364ae3f436a63005877674cf45efb",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}
--- a/flake.nix
+++ b/flake.nix
@@ -0,0 +1,13 @@
+{
+  description = "Panoptikon - Website and command output monitoring";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+  };
+
+  outputs = inputs: {
+    nixosModules.default = ./module.nix;
+    overlays.default = ./overlay.nix;
+  };
+}
+
--- a/module.nix
+++ b/module.nix
@@ -0,0 +1,123 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  options.services.panoptikon = {
+    enable = lib.mkEnableOption "Generic command output / website watcher";
+    watchers = lib.mkOption {
+      type = lib.types.attrsOf (
+        lib.types.submodule (watcher: {
+          options = {
+            script = lib.mkOption {
+              type = lib.types.path;
+              description = ''
+                A script whose stdout is to be watched.
+              '';
+              example = ''
+                pkgs.writers.writeDash "github-meta" '''
+                  ''${pkgs.curl}/bin/curl -sSL https://api.github.com/meta | ''${pkgs.jq}/bin/jq
+                '''
+              '';
+            };
+            frequency = lib.mkOption {
+              type = lib.types.str;
+              description = ''
+                How often to run the script. See systemd.time(7) for more information about the format.
+              '';
+              example = "*:0/3";
+              default = "daily";
+            };
+            loadCredential = lib.mkOption {
+              type = lib.types.listOf lib.types.str;
+              description = ''
+                This can be used to pass secrets to the systemd service without adding them to the nix store.
+              '';
+              default = [ ];
+            };
+            reporters = lib.mkOption {
+              type = lib.types.listOf lib.types.path;
+              description = ''
+                A list of scripts that take the diff (if any) via stdin and report it (e.g. to IRC, Telegram or Prometheus). The name of the watcher will be in the $PANOPTIKON_WATCHER environment variable.
+              '';
+              example = ''
+                [
+                  (pkgs.writers.writeDash "telegram-reporter" '''
+                    ''${pkgs.curl}/bin/curl -X POST https://api.telegram.org/bot''${TOKEN}/sendMessage \
+                      -d chat_id=123456 \
+                      -d text="$(cat)"
+                  ''')
+                  (pkgs.writers.writeDash "notify" '''
+                    ''${pkgs.libnotify}/bin/notify-send "$PANOPTIKON_WATCHER has changed."
+                  ''')
+                ]
+              '';
+            };
+          };
+          config = { };
+        })
+      );
+    };
+  };
+
+  config =
+    let
+      cfg = config.services.panoptikon;
+    in
+    lib.mkIf cfg.enable {
+      users.extraUsers.panoptikon = {
+        isSystemUser = true;
+        createHome = true;
+        home = "/var/lib/panoptikon";
+        group = "panoptikon";
+      };
+
+      users.extraGroups.panoptikon = { };
+
+      systemd.timers = lib.attrsets.mapAttrs' (
+        watcherName: _:
+        lib.nameValuePair "panoptikon-${watcherName}" {
+          timerConfig.RandomizedDelaySec = toString (60 * 60);
+        }
+      ) cfg.watchers;
+
+      systemd.services = lib.attrsets.mapAttrs' (
+        watcherName: watcherOptions:
+        lib.nameValuePair "panoptikon-${watcherName}" {
+          enable = true;
+          startAt = watcherOptions.frequency;
+          serviceConfig = {
+            Type = "oneshot";
+            User = "panoptikon";
+            Group = "panoptikon";
+            WorkingDirectory = "/var/lib/panoptikon";
+            RestartSec = toString (60 * 60);
+            Restart = "on-failure";
+            LoadCredential = watcherOptions.loadCredential;
+          };
+          unitConfig = {
+            StartLimitIntervalSec = "300";
+            StartLimitBurst = "5";
+          };
+          environment.PANOPTIKON_WATCHER = watcherName;
+          wants = [ "network-online.target" ];
+          script = ''
+            set -fux
+            ${watcherOptions.script} > ${lib.escapeShellArg watcherName}
+            diff_output=$(${pkgs.diffutils}/bin/diff --new-file ${
+              lib.escapeShellArg (watcherName + ".old")
+            } ${lib.escapeShellArg watcherName} || :)
+            if [ -n "$diff_output" ]
+            then
+              ${lib.strings.concatMapStringsSep "\n" (
+                reporter: ''echo "$diff_output" | ${reporter} || :''
+              ) watcherOptions.reporters}
+            fi
+            mv ${lib.escapeShellArg watcherName} ${lib.escapeShellArg (watcherName + ".old")}
+          '';
+        }
+      ) cfg.watchers;
+    };
+}
--- a/overlay.nix
+++ b/overlay.nix
@@ -0,0 +1,47 @@
+final: prev: {
+  panoptikon =
+    let
+      lib = prev.lib;
+    in
+    lib.recursiveUpdate prev.panoptikon {
+      url =
+        address:
+        prev.writers.writeDash "watch-url" ''
+          ${prev.curl}/bin/curl -sSL ${lib.escapeShellArg address} \
+            | ${prev.python3Packages.html2text}/bin/html2text --decode-errors=ignore
+        '';
+      urlSelector =
+        selector: address:
+        prev.writers.writeDash "watch-url-selector" ''
+          ${prev.curl}/bin/curl -sSL ${lib.escapeShellArg address} \
+            | ${prev.htmlq}/bin/htmlq ${lib.escapeShellArg selector} \
+            | ${prev.python3Packages.html2text}/bin/html2text
+        '';
+      urlJSON =
+        {
+          jqScript ? ".",
+        }:
+        address:
+        prev.writers.writeDash "watch-url-json" ''
+          ${prev.curl}/bin/curl -sSL ${lib.escapeShellArg address} | ${prev.jq}/bin/jq -f ${prev.writeText "script.jq" jqScript}
+        '';
+
+      # reporter scripts
+      kpaste-irc =
+        {
+          target,
+          retiolumLink ? false,
+          server ? "irc.r",
+          messagePrefix ? "change detected: ",
+          nick ? ''"$PANOPTIKON_WATCHER"-watcher'',
+        }:
+        prev.writers.writeDash "kpaste-irc-reporter" ''
+          KPASTE_CONTENT_TYPE=text/plain ${prev.kpaste}/bin/kpaste \
+            | ${prev.gnused}/bin/sed -n "${if retiolumLink then "2" else "3"}s/^/${messagePrefix}/p" \
+            | ${prev.nur.repos.mic92.ircsink}/bin/ircsink \
+              --nick ${nick} \
+              --server ${server} \
+              --target ${target}
+        '';
+    };
+}