feat: replace urlwatch by panoptikon

modules/panoptikon.nix

@@ -93,6 +93,7 @@
                 ${pkgs.git}/bin/git diff HEAD^ -- ${watcherName} | ${reporter}
               '')
               watcherOptions.reporters}
+              :
             fi
           '';
         })

packages/urlwatch-insecure.patch

@@ -1,12 +0,0 @@
-diff --git a/lib/urlwatch/storage.py b/lib/urlwatch/storage.py
-index 3f6160f..03979f0 100644
---- a/lib/urlwatch/storage.py
-+++ b/lib/urlwatch/storage.py
-@@ -292,7 +292,6 @@ class UrlsBaseFileStorage(BaseTextualFileStorage, metaclass=ABCMeta):
-         shelljob_errors = self.shelljob_security_checks()
-         if shelljob_errors and any(is_shell_job(job) for job in jobs):
-             print(('Removing shell jobs, because %s' % (' and '.join(shelljob_errors),)))
--            jobs = [job for job in jobs if not is_shell_job(job)]
- 
-         return jobs
- 

systems/ful/panoptikon.nix

@@ -18,6 +18,13 @@
       ${pkgs.curl}/bin/curl -sSL ${lib.escapeShellArg address} | ${pkgs.jq}/bin/jq
     '';
 
+  urlSelector = selector: address:
+    pkgs.writers.writeDash "watch-url-selector" ''
+      ${pkgs.curl}/bin/curl -sSL ${lib.escapeShellArg address} \
+        | ${pkgs.htmlq}/bin/htmlq ${lib.escapeShellArg selector} \
+        | ${pkgs.python3Packages.html2text}/bin/html2text
+    '';
+
   reporters.irc-xxx = pkgs.writers.writeDash "irc-xxx" ''
     ${kpaste} \
     | ${pkgs.gnused}/bin/sed -n '2s/^/change detected: /p' \
@@ -26,16 +33,87 @@
       --server irc.r \
       --target '#xxx'
   '';
+
+  reporters.irc-kmein = pkgs.writers.writeDash "irc-xxx" ''
+    ${kpaste} \
+    | ${pkgs.gnused}/bin/sed -n "3s/^/$PANOPTIKON_WATCHER: /p" \
+    | ${config.nur.repos.mic92.ircsink}/bin/ircsink \
+      --nick panoptikon-kmein \
+      --server irc.r \
+      --target 'kmein'
+  '';
 in {
   services.panoptikon = {
     enable = true;
-    watchers."github-meta" = {
+    watchers = {
-      script = urlJSON "https://api.github.com/meta";
+      "github-meta" = {
-      reporters = [reporters.irc-xxx];
+        script = urlJSON "https://api.github.com/meta";
-    };
+        reporters = [reporters.irc-xxx];
-    watchers.spiegel = {
+      };
-      script = url "https://www.spiegel.de/";
+      lammla = {
-      reporters = [];
+        script = url "http://lammla.info/index.php?reihe=30";
+        reporters = [reporters.irc-kmein];
+      };
+      kratylos = {
+        script = url "https://kratylos.reichert-online.org/current_issue/KRATYLOS";
+        reporters = [reporters.irc-kmein];
+      };
+      zeno-free = {
+        script = urlSelector ".zenoCOMain" "http://www.zeno.org/Lesesaal/M/E-Books";
+        reporters = [reporters.irc-kmein];
+      };
+      carolinawelslau = {
+        script = urlSelector "#main" "https://carolinawelslau.de/";
+        reporters = [reporters.irc-kmein];
+      };
+      lisalittmann = {
+        script = urlSelector "#main" "https://lisalittmann.de/";
+        reporters = [reporters.irc-kmein];
+      };
+      lisalittmann-archive = {
+        script = urlSelector "#main" "https://lisalittmann.de/archive/";
+        reporters = [reporters.irc-kmein];
+      };
+      lisalittmann-projects = {
+        script = urlSelector "#main" "https://lisalittmann.de/projects/";
+        reporters = [reporters.irc-kmein];
+      };
+      tatort = {
+        script = urlSelector ".linklist" "https://www.daserste.de/unterhaltung/krimi/tatort/sendung/index.html";
+        reporters = [reporters.irc-kmein];
+      };
+      warpgrid-idiomarium = {
+        script = urlSelector "#site-content" "https://warpgrid.de/idiomarium/";
+        reporters = [reporters.irc-kmein];
+      };
+      warpgrid-futurism = {
+        script = urlSelector "#site-content" "https://warpgrid.de/futurism/";
+        reporters = [reporters.irc-kmein];
+      };
+      warpgrid-imagiary = {
+        script = urlSelector "#site-content" "https://warpgrid.de/imagiary/";
+        reporters = [reporters.irc-kmein];
+      };
+      warpgrid-alchemy = {
+        script = urlSelector "#site-content" "https://warpgrid.de/alchemy/";
+        reporters = [reporters.irc-kmein];
+      };
+      indogermanische-forschungen = {
+        script = urlSelector "#latestIssue" "https://www.degruyter.com/journal/key/INDO/html";
+        reporters = [reporters.irc-kmein];
+      };
+      ig-neuigkeiten = {
+        script = urlSelector "[itemprop=articleBody]" "https://www.indogermanistik.org/aktuelles/neuigkeiten.html";
+        reporters = [reporters.irc-kmein];
+      };
+      ig-tagungen = {
+        script = urlSelector "[itemprop=articleBody]" "https://www.indogermanistik.org/aktuelles/tagungen-der-ig.html";
+        reporters = [reporters.irc-kmein];
+      };
+      fxght-or-flxght = {
+        script = urlJSON "https://api.tellonym.me/profiles/name/fxght.or.flxght?limit=20";
+        reporters = [reporters.irc-kmein];
+      };
     };
   };
 }

systems/makanek/configuration.nix

@@ -19,7 +19,6 @@ in {
     ./retiolum-map.nix
     ./tarot.nix
     ./tt-rss.nix
-    ./urlwatch.nix
     ./weechat.nix
     ../../configs/monitoring.nix
     ../../configs/nix.nix

systems/makanek/urlwatch.nix

@@ -1,204 +0,0 @@
-{
-  pkgs,
-  config,
-  lib,
-  ...
-}: let
-  inherit (import ../../lib) kieran;
-
-  urlwatchDir = "/var/lib/urlwatch";
-
-  urlsFile = pkgs.writeText "urls" (builtins.concatStringsSep "\n---\n" (map builtins.toJSON urls));
-
-  tokensFile = config.age.secrets.urlwatch-tokens.path;
-
-  urls = [
-    {
-      name = "Corona-Verordnung";
-      url = "https://www.berlin.de/corona/massnahmen/verordnung/";
-      filter = [{css = "[role=main]";} "html2text" "strip"];
-    }
-    {
-      name = "HU Semester";
-      url = "https://agnes.hu-berlin.de/lupo/rds?state=change&type=6&moduleParameter=semesterSelect&nextdir=change&next=SearchSelect.vm&subdir=applications&targettype=7&targetstate=change&getglobal=semester";
-      filter = [{css = "fieldset";} "html2text" "strip"];
-    }
-    {
-      name = "Lammla 2021";
-      url = "http://lammla.info/index.php?reihe=30";
-      filter = ["html2text" "strip"];
-    }
-    {
-      name = "Kratylos";
-      url = "https://kratylos.reichert-online.org/current_issue/KRATYLOS";
-      filter = [{element-by-id = "content";} "html2text" "strip"];
-    }
-    {
-      name = "Zeno Free E-Books";
-      url = "http://www.zeno.org/Lesesaal/M/E-Books";
-      filter = [{element-by-class = "zenoCOMain";} "html2text" "strip"];
-    }
-    {
-      name = "Carolina Welslau";
-      url = "https://carolinawelslau.de/";
-      filter = [{element-by-id = "main";} "html2text" "strip"];
-    }
-    {
-      name = "Lisa Littmann";
-      url = "https://lisalittmann.de/";
-      filter = [{element-by-id = "main";} "html2text" "strip"];
-    }
-    {
-      name = "Tatort: Fälle";
-      url = "https://www.daserste.de/unterhaltung/krimi/tatort/sendung/index.html";
-      filter = [{element-by-class = "linklist";} "html2text" "strip"];
-    }
-    {
-      name = "Lisa Littmann: Projects";
-      url = "https://lisalittmann.de/projects/";
-      filter = [{element-by-id = "main";} "html2text" "strip"];
-    }
-    {
-      name = "Lisa Littmann: Archive";
-      url = "https://lisalittmann.de/archive/";
-      filter = [{element-by-id = "main";} "html2text" "strip"];
-    }
-    {
-      name = "WarpGrid: Idiomarium";
-      url = "https://warpgrid.de/idiomarium/";
-      filter = [{element-by-id = "site-content";} "html2text" "strip"];
-    }
-    {
-      name = "WarpGrid: Futurism";
-      url = "https://warpgrid.de/futurism/";
-      filter = [{element-by-id = "site-content";} "html2text" "strip"];
-    }
-    {
-      name = "WarpGrid: Imagiary";
-      url = "https://warpgrid.de/imagiary/";
-      filter = [{element-by-id = "site-content";} "html2text" "strip"];
-    }
-    {
-      name = "WarpGrid: Cook";
-      url = "https://warpgrid.de/alchemy/";
-      filter = [{element-by-id = "site-content";} "html2text" "strip"];
-    }
-    {
-      name = "Indogermanische Forschungen";
-      url = "https://www.degruyter.com/journal/key/INDO/html";
-      filter = [{element-by-id = "latestIssue";} "html2text" "strip"];
-    }
-    {
-      name = "IG Neuigkeiten";
-      url = "https://www.indogermanistik.org/aktuelles/neuigkeiten.html";
-      filter = [{css = "[itemprop=articleBody]";} "html2text" "strip"];
-    }
-    {
-      name = "IG Tagungen";
-      url = "https://www.indogermanistik.org/tagungen/tagungen-der-ig.html";
-      filter = [{css = "[itemprop=articleBody]";} "html2text" "strip"];
-    }
-    {
-      name = "Christian-Metz-Blamage";
-      url = "https://www.deutschlandfunk.de/meine-nacht-schlaeft-nicht-pflanze-mich-nicht-in-dein-herz.700.de.html?dram:article_id=486475";
-      filter = [{element-by-class = "dlf-articledetail";} "html2text" "strip"];
-    }
-    {
-      name = "Ist der Congress schon abgesagt?";
-      url = "https://ist-der-congress-schon-abgesagt.de/";
-    }
-    {
-      name = "fxght.or.flxght";
-      url = "https://api.tellonym.me/profiles/name/fxght.or.flxght?limit=20";
-      headers.tellonym-client = "web:0.52.0";
-      filter = [
-        {
-          shellpipe = ''
-            ${pkgs.jq}/bin/jq '.answers | map({
-              question: .tell,
-              answer: .answer,
-              date: .createdAt,
-              media: .media | map(.url)
-            })'
-          '';
-        }
-      ];
-    }
-  ];
-
-  configFile = (pkgs.formats.json {}).generate "urlwatch.json" {
-    display = {
-      error = true;
-      new = true;
-      unchanged = false;
-    };
-    report = {
-      email = {
-        enabled = true;
-        from = "2210@cock.li";
-        html = false;
-        method = "smtp";
-        smtp = {
-          host = "mail.cock.li";
-          port = 587;
-          starttls = true;
-          auth = true;
-        };
-        subject = "{count} changes: {jobs}";
-        to = kieran.email;
-      };
-      telegram = {
-        enabled = true;
-        chat_id = "-1001504043752";
-      };
-      html.diff = "unified";
-      stdout = {
-        color = true;
-        enabled = true;
-      };
-      text.footer = false;
-    };
-  };
-  urlwatch = pkgs.urlwatch.overrideAttrs (attrs: {
-    patches = [../../packages/urlwatch-insecure.patch];
-  });
-in {
-  users.extraUsers.urlwatch = {
-    home = urlwatchDir;
-    createHome = true;
-    isSystemUser = true;
-    group = "urlwatch";
-  };
-
-  age.secrets.urlwatch-tokens.file = ../../secrets/urlwatch-tokens.json.age;
-
-  users.groups.urlwatch = {};
-
-  systemd.services.urlwatch = {
-    enable = true;
-    startAt = "12:00";
-    script = ''
-      ${urlwatch}/bin/urlwatch \
-        --config=<(
-          ${pkgs.jq}/bin/jq -s '.[0] * .[1]' ${toString configFile} ${toString tokensFile}
-        ) \
-        --urls=${lib.escapeShellArg urlsFile}
-    '';
-    serviceConfig = {
-      User = config.users.extraUsers.urlwatch.name;
-      Group = config.users.groups.urlwatch.name;
-      WorkingDirectory = config.users.extraUsers.urlwatch.home;
-      PermissionsStartOnly = "true";
-      PrivateTmp = "true";
-      SyslogIdentifier = "urlwatch";
-      Type = "oneshot";
-    };
-  };
-
-  niveum.passport.services = [
-    {
-      description = "keeps me up-to-date on sites that have no RSS feed (shame be upon them!).";
-      title = "urlwatch";
-    }
-  ];
-}