231 lines
8.4 KiB
Org Mode
231 lines
8.4 KiB
Org Mode
:PROPERTIES:
|
|
:ID: 20221227T164309.780458
|
|
:ROAM_ALIASES: feed2toot
|
|
:ROAM_REFS: https://gitlab.com/chaica/feed2toot https://feed2toot.readthedocs.io/en/latest/
|
|
:END:
|
|
#+TITLE: Posting Arcology Feeds to the Fediverse Automatically with feed2toot (deprecated)
|
|
#+filetags: :CCE:Fediverse:
|
|
|
|
#+ARCOLOGY_KEY: arcology/feed2toot
|
|
#+ARCOLOGY_ALLOW_CRAWL: t
|
|
|
|
#+AUTO_TANGLE: t
|
|
|
|
#+begin_quote
|
|
Feed2toot automatically parses rss feeds, identifies new posts and posts them on the +Mastodon+ [[id:62538db5-d94a-47c3-9998-086ded91fd88][Fediverse]] social network.
|
|
#+end_quote
|
|
|
|
I'll probably end up writing something similar myself one of these days since (somewhat critically) this does not support multiple (feeds->account) mappings right now and thus I will have to run one for each of my [[id:20211219T144255.001827][Arcology Sites]]' feeds.
|
|
|
|
* =feed2toot= [[id:20221021T121120.541960][rixpkgs]] entry
|
|
|
|
We love a Python package that is pretty easy to get running:
|
|
|
|
#+begin_src nix :tangle ~/arroyo-nix/pkgs/feed2toot.nix
|
|
{ lib,
|
|
buildPythonPackage,
|
|
fetchPypi,
|
|
beautifulsoup4,
|
|
feedparser,
|
|
mastodon-py,
|
|
python,
|
|
}:
|
|
|
|
buildPythonPackage rec {
|
|
pname = "feed2toot";
|
|
version = "0.17";
|
|
|
|
src = fetchPypi {
|
|
inherit pname version;
|
|
sha256 = "sha256-VKj0Wu5KdsfsywO1g4z3mFsQT9Svrt6wQHZSeNxbRAw=";
|
|
};
|
|
|
|
propagatedBuildInputs = [
|
|
beautifulsoup4
|
|
feedparser
|
|
mastodon-py
|
|
python.pkgs.influxdb
|
|
];
|
|
|
|
meta = with lib; {
|
|
homepage = "https://feed2toot.readthedocs.io/en/latest/";
|
|
description = "Feed2toot parses a RSS feed, extracts the last entries and sends them to Mastodon.";
|
|
license = licenses.gpl3;
|
|
maintainers = with maintainers; [ rrix ];
|
|
};
|
|
}
|
|
#+end_src
|
|
|
|
[[shell:nix-build '<arroyo>' -A feed2toot]] -> /nix/store/fvw5m7q8y2kh0mkr2y52wv55lvv2hwym-python3.10-feed2toot-0.17
|
|
|
|
* Generating the Feed URI Lists Programmatically
|
|
|
|
So I have a few [[id:20211219T144255.001827][Arcology Sites]], each of these needs its own configuration file passed to the =feed2toot= invocation. This queries [[id:arcology/arroyo-page][Arroyo Arcology Generator]]'s [[id:arcology/arroyo/feed][arcology.arroyo.Feed]] to generate a file with each group of =(site, visibility)= of feeds in the database.
|
|
|
|
#+NAME: site-filter
|
|
#+begin_src emacs-lisp :var site-filter="lionsrear" :var site-domain="https://thelionsrear.com/" :results drawer
|
|
(setq arcology-sites '(("lionsrear" . "https://thelionsrear.com/")
|
|
("garden" . "https://arcology.garden/")
|
|
("cce" . "https://cce.whatthefuck.computer/")
|
|
("arcology" . "https://engine.arcology.garden/")))
|
|
|
|
(defun arcology-arroyo-feeds-to-urls (group rows)
|
|
(let* ((file-name (format "~/arroyo-nix/files/feed2toot-uris-%s-%s.txt" (first group) (second group))))
|
|
(list file-name
|
|
(-map (lambda (row)
|
|
(let* ((feed-prefix (alist-get (nth 3 row) arcology-sites nil nil #'equal))
|
|
(site (nth 3 row))
|
|
(key (nth 1 row))
|
|
(feed-url (replace-regexp-in-string (concat site "/") feed-prefix key)))
|
|
feed-url
|
|
)
|
|
)
|
|
rows))))
|
|
|
|
(defun arcology-arroyo-write-feed-list (file lines)
|
|
(with-temp-buffer
|
|
(insert (s-join "\n" lines))
|
|
(write-file file)
|
|
file))
|
|
|
|
(defun arcology-arroyo-make-feed-lists ()
|
|
(->> (arroyo-db-query [:select * :from arcology-feeds])
|
|
(-group-by (lambda (row)
|
|
(list (nth 3 row) (nth 4 row))))
|
|
(-map (lambda (partitioned)
|
|
(let* ((partition (first partitioned))
|
|
(rows (cdr partitioned)))
|
|
(arcology-arroyo-feeds-to-urls partition rows))))
|
|
(-map (lambda (group)
|
|
(apply #'arcology-arroyo-write-feed-list group)))))
|
|
|
|
(s-join "\n" (arcology-arroyo-make-feed-lists))
|
|
#+end_src
|
|
|
|
#+results: site-filter
|
|
:results:
|
|
~/arroyo-nix/files/feed2toot-uris-garden-private.txt
|
|
~/arroyo-nix/files/feed2toot-uris-arcology-public.txt
|
|
~/arroyo-nix/files/feed2toot-uris-garden-public.txt
|
|
~/arroyo-nix/files/feed2toot-uris-lionsrear-unlisted.txt
|
|
~/arroyo-nix/files/feed2toot-uris-lionsrear-public.txt
|
|
~/arroyo-nix/files/feed2toot-uris-cce-public.txt
|
|
:end:
|
|
|
|
* INPROGRESS deploy to [[id:20211120T220054.226284][The Wobserver]]
|
|
:LOGBOOK:
|
|
- State "INPROGRESS" from "NEXT" [2022-12-28 Wed 12:38]
|
|
:END:
|
|
=feed2toot= operates with [[https://feed2toot.readthedocs.io/en/latest/configure.html#create-feed2toot-configuration][INI configuration files]] and luckily enough =nixpkgs= includes a way to convert =attrsets= to =ini= files in =lib.generators.toINI=; this is wrapped in a function which takes the basic per-instance configuration details including the uri-list which are generated above and then creates a [[roam:SystemD]] service manifest and timer to poll the site and post the toots. Easy peazy.
|
|
|
|
,#+ARROYO_NIXOS_MODULE: nixos/feed2toot.nix
|
|
,#+ARROYO_SYSTEM_ROLE: server
|
|
|
|
#+begin_src nix :tangle ~/arroyo-nix/nixos/feed2toot.nix :noweb yes
|
|
{ pkgs, lib, config, ... }:
|
|
|
|
let
|
|
mkFeed2TootIni = { instance,
|
|
uriList,
|
|
visibility ? "public",
|
|
credLoc ? config.users.users.feed2toot.home,
|
|
tagListFile ? pkgs.writeText "empty-taglist" ""}:
|
|
pkgs.writeText "feed2toot-${instance}"
|
|
(lib.generators.toINI {}
|
|
{
|
|
mastodon = {
|
|
instance_url = "https://notes.whatthefuck.computer";
|
|
user_credentials = "${credLoc}/${instance}_feed2toot_usercred.txt";
|
|
client_credentials = "${credLoc}/${instance}_feed2toot_clientcred.txt";
|
|
toot_visibility = visibility;
|
|
};
|
|
cache.cachefile = "/var/cache/feed2toot/${instance}.db";
|
|
lock.lock_file = "/run/feed2toot/${instance}.lock";
|
|
rss.uri_list = builtins.path { path = uriList; };
|
|
rss.toot = "NEW by @rrix@notes.whatthefuck.computer: {link} \\n {summary}";
|
|
rss.toot_max_len = 10000;
|
|
hashtaglist.several_words_hashtags_list = tagListFile;
|
|
feedparser.accept_bozo_exceptions = true;
|
|
});
|
|
|
|
feeds = [
|
|
(mkFeed2TootIni {
|
|
instance = "garden";
|
|
visibility = "public";
|
|
uriList = ../files/feed2toot-uris-cce-public.txt;
|
|
})
|
|
(mkFeed2TootIni {
|
|
instance = "garden";
|
|
visibility = "public";
|
|
uriList = ../files/feed2toot-uris-garden-public.txt;
|
|
})
|
|
(mkFeed2TootIni {
|
|
instance = "lionsrear";
|
|
visibility = "public";
|
|
uriList = ../files/feed2toot-uris-lionsrear-public.txt;
|
|
})
|
|
(mkFeed2TootIni {
|
|
instance = "garden";
|
|
visibility = "unlisted";
|
|
uriList = ../files/feed2toot-uris-arcology-public.txt;
|
|
})
|
|
(mkFeed2TootIni {
|
|
instance = "lionsrear";
|
|
visibility = "unlisted";
|
|
uriList = ../files/feed2toot-uris-lionsrear-unlisted.txt;
|
|
})
|
|
(mkFeed2TootIni {
|
|
instance = "garden";
|
|
visibility = "private";
|
|
uriList = ../files/feed2toot-uris-garden-private.txt;
|
|
})
|
|
];
|
|
in {
|
|
ids.uids.feed2toot = 902;
|
|
ids.gids.bots = 902;
|
|
|
|
users.groups.bots = {
|
|
gid = config.ids.gids.bots;
|
|
};
|
|
users.users."feed2toot" = {
|
|
home = "/srv/feed2toot/";
|
|
group = "bots";
|
|
uid = config.ids.uids.feed2toot;
|
|
isSystemUser = true;
|
|
};
|
|
|
|
systemd.tmpfiles.rules = [
|
|
"d /run/feed2toot 1777 feed2toot bots" # lock file directory
|
|
"d /var/cache/feed2toot 1777 feed2toot bots" # cache file directory
|
|
"d /srv/feed2toot 1777 feed2toot bots" # working directory
|
|
];
|
|
|
|
systemd.services.feed2toot = {
|
|
description = "Feeds to Toots";
|
|
after = ["pleroma.service"];
|
|
wantedBy = ["default.target"];
|
|
script =
|
|
lib.concatMapStrings
|
|
(feed: "\n${pkgs.feed2toot}/bin/feed2toot -c " + feed)
|
|
feeds;
|
|
serviceConfig = {
|
|
User = "feed2toot";
|
|
WorkingDirectory = "/srv/feed2toot";
|
|
};
|
|
};
|
|
systemd.timers.feed2toot = {
|
|
description = "Start feed2toot on the quarter-hour";
|
|
timerConfig = {
|
|
OnUnitActiveSec = "15 minutes";
|
|
OnStartupSec = "15 minutes";
|
|
};
|
|
wantedBy = [ "default.target" ];
|
|
};
|
|
}
|
|
#+end_src
|
|
|
|
* NEXT remove bs4 html filtering
|
|
|
|
pleroma supports embedding HTML so we should use that.
|