From 29985fffc278c61e23704fee908f7cb4f9514441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kier=C3=A1n=20Meinhardt?= Date: Fri, 24 Mar 2023 19:02:18 +0100 Subject: [PATCH] feat: replace urlwatch by panoptikon --- modules/panoptikon.nix | 1 + packages/urlwatch-insecure.patch | 12 -- systems/ful/panoptikon.nix | 92 +++++++++++++- systems/makanek/configuration.nix | 1 - systems/makanek/urlwatch.nix | 204 ------------------------------ 5 files changed, 86 insertions(+), 224 deletions(-) delete mode 100644 packages/urlwatch-insecure.patch delete mode 100644 systems/makanek/urlwatch.nix diff --git a/modules/panoptikon.nix b/modules/panoptikon.nix index ae82261..ab9a9fc 100644 --- a/modules/panoptikon.nix +++ b/modules/panoptikon.nix @@ -93,6 +93,7 @@ ${pkgs.git}/bin/git diff HEAD^ -- ${watcherName} | ${reporter} '') watcherOptions.reporters} + : fi ''; }) diff --git a/packages/urlwatch-insecure.patch b/packages/urlwatch-insecure.patch deleted file mode 100644 index 67d2583..0000000 --- a/packages/urlwatch-insecure.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/lib/urlwatch/storage.py b/lib/urlwatch/storage.py -index 3f6160f..03979f0 100644 ---- a/lib/urlwatch/storage.py -+++ b/lib/urlwatch/storage.py -@@ -292,7 +292,6 @@ class UrlsBaseFileStorage(BaseTextualFileStorage, metaclass=ABCMeta): - shelljob_errors = self.shelljob_security_checks() - if shelljob_errors and any(is_shell_job(job) for job in jobs): - print(('Removing shell jobs, because %s' % (' and '.join(shelljob_errors),))) -- jobs = [job for job in jobs if not is_shell_job(job)] - - return jobs - diff --git a/systems/ful/panoptikon.nix b/systems/ful/panoptikon.nix index d37b694..f3896eb 100644 --- a/systems/ful/panoptikon.nix +++ b/systems/ful/panoptikon.nix @@ -18,6 +18,13 @@ ${pkgs.curl}/bin/curl -sSL ${lib.escapeShellArg address} | ${pkgs.jq}/bin/jq ''; + urlSelector = selector: address: + pkgs.writers.writeDash "watch-url-selector" '' + ${pkgs.curl}/bin/curl -sSL ${lib.escapeShellArg address} \ + | ${pkgs.htmlq}/bin/htmlq ${lib.escapeShellArg selector} \ + | ${pkgs.python3Packages.html2text}/bin/html2text + ''; + reporters.irc-xxx = pkgs.writers.writeDash "irc-xxx" '' ${kpaste} \ | ${pkgs.gnused}/bin/sed -n '2s/^/change detected: /p' \ @@ -26,16 +33,87 @@ --server irc.r \ --target '#xxx' ''; + + reporters.irc-kmein = pkgs.writers.writeDash "irc-xxx" '' + ${kpaste} \ + | ${pkgs.gnused}/bin/sed -n "3s/^/$PANOPTIKON_WATCHER: /p" \ + | ${config.nur.repos.mic92.ircsink}/bin/ircsink \ + --nick panoptikon-kmein \ + --server irc.r \ + --target 'kmein' + ''; in { services.panoptikon = { enable = true; - watchers."github-meta" = { - script = urlJSON "https://api.github.com/meta"; - reporters = [reporters.irc-xxx]; - }; - watchers.spiegel = { - script = url "https://www.spiegel.de/"; - reporters = []; + watchers = { + "github-meta" = { + script = urlJSON "https://api.github.com/meta"; + reporters = [reporters.irc-xxx]; + }; + lammla = { + script = url "http://lammla.info/index.php?reihe=30"; + reporters = [reporters.irc-kmein]; + }; + kratylos = { + script = url "https://kratylos.reichert-online.org/current_issue/KRATYLOS"; + reporters = [reporters.irc-kmein]; + }; + zeno-free = { + script = urlSelector ".zenoCOMain" "http://www.zeno.org/Lesesaal/M/E-Books"; + reporters = [reporters.irc-kmein]; + }; + carolinawelslau = { + script = urlSelector "#main" "https://carolinawelslau.de/"; + reporters = [reporters.irc-kmein]; + }; + lisalittmann = { + script = urlSelector "#main" "https://lisalittmann.de/"; + reporters = [reporters.irc-kmein]; + }; + lisalittmann-archive = { + script = urlSelector "#main" "https://lisalittmann.de/archive/"; + reporters = [reporters.irc-kmein]; + }; + lisalittmann-projects = { + script = urlSelector "#main" "https://lisalittmann.de/projects/"; + reporters = [reporters.irc-kmein]; + }; + tatort = { + script = urlSelector ".linklist" "https://www.daserste.de/unterhaltung/krimi/tatort/sendung/index.html"; + reporters = [reporters.irc-kmein]; + }; + warpgrid-idiomarium = { + script = urlSelector "#site-content" "https://warpgrid.de/idiomarium/"; + reporters = [reporters.irc-kmein]; + }; + warpgrid-futurism = { + script = urlSelector "#site-content" "https://warpgrid.de/futurism/"; + reporters = [reporters.irc-kmein]; + }; + warpgrid-imagiary = { + script = urlSelector "#site-content" "https://warpgrid.de/imagiary/"; + reporters = [reporters.irc-kmein]; + }; + warpgrid-alchemy = { + script = urlSelector "#site-content" "https://warpgrid.de/alchemy/"; + reporters = [reporters.irc-kmein]; + }; + indogermanische-forschungen = { + script = urlSelector "#latestIssue" "https://www.degruyter.com/journal/key/INDO/html"; + reporters = [reporters.irc-kmein]; + }; + ig-neuigkeiten = { + script = urlSelector "[itemprop=articleBody]" "https://www.indogermanistik.org/aktuelles/neuigkeiten.html"; + reporters = [reporters.irc-kmein]; + }; + ig-tagungen = { + script = urlSelector "[itemprop=articleBody]" "https://www.indogermanistik.org/aktuelles/tagungen-der-ig.html"; + reporters = [reporters.irc-kmein]; + }; + fxght-or-flxght = { + script = urlJSON "https://api.tellonym.me/profiles/name/fxght.or.flxght?limit=20"; + reporters = [reporters.irc-kmein]; + }; }; }; } diff --git a/systems/makanek/configuration.nix b/systems/makanek/configuration.nix index 321526a..767d809 100644 --- a/systems/makanek/configuration.nix +++ b/systems/makanek/configuration.nix @@ -19,7 +19,6 @@ in { ./retiolum-map.nix ./tarot.nix ./tt-rss.nix - ./urlwatch.nix ./weechat.nix ../../configs/monitoring.nix ../../configs/nix.nix diff --git a/systems/makanek/urlwatch.nix b/systems/makanek/urlwatch.nix deleted file mode 100644 index a7bd736..0000000 --- a/systems/makanek/urlwatch.nix +++ /dev/null @@ -1,204 +0,0 @@ -{ - pkgs, - config, - lib, - ... -}: let - inherit (import ../../lib) kieran; - - urlwatchDir = "/var/lib/urlwatch"; - - urlsFile = pkgs.writeText "urls" (builtins.concatStringsSep "\n---\n" (map builtins.toJSON urls)); - - tokensFile = config.age.secrets.urlwatch-tokens.path; - - urls = [ - { - name = "Corona-Verordnung"; - url = "https://www.berlin.de/corona/massnahmen/verordnung/"; - filter = [{css = "[role=main]";} "html2text" "strip"]; - } - { - name = "HU Semester"; - url = "https://agnes.hu-berlin.de/lupo/rds?state=change&type=6&moduleParameter=semesterSelect&nextdir=change&next=SearchSelect.vm&subdir=applications&targettype=7&targetstate=change&getglobal=semester"; - filter = [{css = "fieldset";} "html2text" "strip"]; - } - { - name = "Lammla 2021"; - url = "http://lammla.info/index.php?reihe=30"; - filter = ["html2text" "strip"]; - } - { - name = "Kratylos"; - url = "https://kratylos.reichert-online.org/current_issue/KRATYLOS"; - filter = [{element-by-id = "content";} "html2text" "strip"]; - } - { - name = "Zeno Free E-Books"; - url = "http://www.zeno.org/Lesesaal/M/E-Books"; - filter = [{element-by-class = "zenoCOMain";} "html2text" "strip"]; - } - { - name = "Carolina Welslau"; - url = "https://carolinawelslau.de/"; - filter = [{element-by-id = "main";} "html2text" "strip"]; - } - { - name = "Lisa Littmann"; - url = "https://lisalittmann.de/"; - filter = [{element-by-id = "main";} "html2text" "strip"]; - } - { - name = "Tatort: Fälle"; - url = "https://www.daserste.de/unterhaltung/krimi/tatort/sendung/index.html"; - filter = [{element-by-class = "linklist";} "html2text" "strip"]; - } - { - name = "Lisa Littmann: Projects"; - url = "https://lisalittmann.de/projects/"; - filter = [{element-by-id = "main";} "html2text" "strip"]; - } - { - name = "Lisa Littmann: Archive"; - url = "https://lisalittmann.de/archive/"; - filter = [{element-by-id = "main";} "html2text" "strip"]; - } - { - name = "WarpGrid: Idiomarium"; - url = "https://warpgrid.de/idiomarium/"; - filter = [{element-by-id = "site-content";} "html2text" "strip"]; - } - { - name = "WarpGrid: Futurism"; - url = "https://warpgrid.de/futurism/"; - filter = [{element-by-id = "site-content";} "html2text" "strip"]; - } - { - name = "WarpGrid: Imagiary"; - url = "https://warpgrid.de/imagiary/"; - filter = [{element-by-id = "site-content";} "html2text" "strip"]; - } - { - name = "WarpGrid: Cook"; - url = "https://warpgrid.de/alchemy/"; - filter = [{element-by-id = "site-content";} "html2text" "strip"]; - } - { - name = "Indogermanische Forschungen"; - url = "https://www.degruyter.com/journal/key/INDO/html"; - filter = [{element-by-id = "latestIssue";} "html2text" "strip"]; - } - { - name = "IG Neuigkeiten"; - url = "https://www.indogermanistik.org/aktuelles/neuigkeiten.html"; - filter = [{css = "[itemprop=articleBody]";} "html2text" "strip"]; - } - { - name = "IG Tagungen"; - url = "https://www.indogermanistik.org/tagungen/tagungen-der-ig.html"; - filter = [{css = "[itemprop=articleBody]";} "html2text" "strip"]; - } - { - name = "Christian-Metz-Blamage"; - url = "https://www.deutschlandfunk.de/meine-nacht-schlaeft-nicht-pflanze-mich-nicht-in-dein-herz.700.de.html?dram:article_id=486475"; - filter = [{element-by-class = "dlf-articledetail";} "html2text" "strip"]; - } - { - name = "Ist der Congress schon abgesagt?"; - url = "https://ist-der-congress-schon-abgesagt.de/"; - } - { - name = "fxght.or.flxght"; - url = "https://api.tellonym.me/profiles/name/fxght.or.flxght?limit=20"; - headers.tellonym-client = "web:0.52.0"; - filter = [ - { - shellpipe = '' - ${pkgs.jq}/bin/jq '.answers | map({ - question: .tell, - answer: .answer, - date: .createdAt, - media: .media | map(.url) - })' - ''; - } - ]; - } - ]; - - configFile = (pkgs.formats.json {}).generate "urlwatch.json" { - display = { - error = true; - new = true; - unchanged = false; - }; - report = { - email = { - enabled = true; - from = "2210@cock.li"; - html = false; - method = "smtp"; - smtp = { - host = "mail.cock.li"; - port = 587; - starttls = true; - auth = true; - }; - subject = "{count} changes: {jobs}"; - to = kieran.email; - }; - telegram = { - enabled = true; - chat_id = "-1001504043752"; - }; - html.diff = "unified"; - stdout = { - color = true; - enabled = true; - }; - text.footer = false; - }; - }; - urlwatch = pkgs.urlwatch.overrideAttrs (attrs: { - patches = [../../packages/urlwatch-insecure.patch]; - }); -in { - users.extraUsers.urlwatch = { - home = urlwatchDir; - createHome = true; - isSystemUser = true; - group = "urlwatch"; - }; - - age.secrets.urlwatch-tokens.file = ../../secrets/urlwatch-tokens.json.age; - - users.groups.urlwatch = {}; - - systemd.services.urlwatch = { - enable = true; - startAt = "12:00"; - script = '' - ${urlwatch}/bin/urlwatch \ - --config=<( - ${pkgs.jq}/bin/jq -s '.[0] * .[1]' ${toString configFile} ${toString tokensFile} - ) \ - --urls=${lib.escapeShellArg urlsFile} - ''; - serviceConfig = { - User = config.users.extraUsers.urlwatch.name; - Group = config.users.groups.urlwatch.name; - WorkingDirectory = config.users.extraUsers.urlwatch.home; - PermissionsStartOnly = "true"; - PrivateTmp = "true"; - SyslogIdentifier = "urlwatch"; - Type = "oneshot"; - }; - }; - - niveum.passport.services = [ - { - description = "keeps me up-to-date on sites that have no RSS feed (shame be upon them!)."; - title = "urlwatch"; - } - ]; -}