add feed2toot for arcology etc
Ryan Rix
5 months ago
parent
b657dc62f6
commit
f629ba2e0e
@ -0,0 +1,209 @@
|
||||
:PROPERTIES:
|
||||
:ID: 20221227T164309.780458
|
||||
:ROAM_ALIASES: feed2toot
|
||||
:ROAM_REFS: https://gitlab.com/chaica/feed2toot https://feed2toot.readthedocs.io/en/latest/
|
||||
:END:
|
||||
#+TITLE: Posting Arcology Feeds to the Fediverse Automatically
|
||||
#+filetags: :Project:CCE:Fediverse:
|
||||
#+ARCOLOGY_KEY: arcology/feed2toot
|
||||
#+ARCOLOGY_ALLOW_CRAWL: t
|
||||
|
||||
#+begin_quote
|
||||
Feed2toot automatically parses rss feeds, identifies new posts and posts them on the +Mastodon+ [[id:62538db5-d94a-47c3-9998-086ded91fd88][Fediverse]] social network.
|
||||
#+end_quote
|
||||
|
||||
I'll probably end up writing something similar myself one of these days since (somewhat critically) this does not support multiple (feeds->account) mappings right now and thus I will have to run one for each of my [[id:20211219T144255.001827][Arcology Sites]]' feeds.
|
||||
|
||||
* =feed2toot= [[id:20221021T121120.541960][rixpkgs]] entry
|
||||
|
||||
We love a Python package that is pretty easy to get running:
|
||||
|
||||
#+begin_src nix :tangle ~/arroyo-nix/pkgs/feed2toot.nix
|
||||
{ lib,
|
||||
buildPythonPackage,
|
||||
fetchPypi,
|
||||
beautifulsoup4,
|
||||
feedparser,
|
||||
mastodon-py,
|
||||
python,
|
||||
}:
|
||||
|
||||
buildPythonPackage rec {
|
||||
pname = "feed2toot";
|
||||
version = "0.17";
|
||||
|
||||
src = fetchPypi {
|
||||
inherit pname version;
|
||||
sha256 = "sha256-VKj0Wu5KdsfsywO1g4z3mFsQT9Svrt6wQHZSeNxbRAw=";
|
||||
};
|
||||
|
||||
propagatedBuildInputs = [
|
||||
beautifulsoup4
|
||||
feedparser
|
||||
mastodon-py
|
||||
python.pkgs.influxdb
|
||||
];
|
||||
|
||||
meta = with lib; {
|
||||
homepage = "https://feed2toot.readthedocs.io/en/latest/";
|
||||
description = "Feed2toot parses a RSS feed, extracts the last entries and sends them to Mastodon.";
|
||||
license = licenses.gpl3;
|
||||
maintainers = with maintainers; [ rrix ];
|
||||
};
|
||||
}
|
||||
#+end_src
|
||||
|
||||
[[shell:nix-build '<arroyo>' -A feed2toot]] -> /nix/store/fvw5m7q8y2kh0mkr2y52wv55lvv2hwym-python3.10-feed2toot-0.17
|
||||
|
||||
* Generating the Feed URI Lists Programmatically
|
||||
|
||||
So I have a few [[id:20211219T144255.001827][Arcology Sites]], each of these needs its own configuration file passed to the =feed2toot= invocation. This queries [[id:arcology/arroyo-page][Arroyo Arcology Generator]]'s [[id:arcology/arroyo/feed][arcology.arroyo.Feed]] to generate a file with each group of =(site, visibility)= of feeds in the database.
|
||||
|
||||
#+NAME: site-filter
|
||||
#+begin_src emacs-lisp :var site-filter="lionsrear" :var site-domain="https://thelionsrear.com/" :results raw
|
||||
(setq arcology-sites '(("lionsrear" . "https://thelionsrear.com/")
|
||||
("garden" . "https://arcology.garden/")
|
||||
("cce" . "https://cce.whatthefuck.computer/")
|
||||
("arcology" . "https://engine.arcology.garden/")))
|
||||
|
||||
(defun arcology-arroyo-feeds-to-urls (group rows)
|
||||
(let* ((file-name (format "~/arroyo-nix/files/feed2toot-uris-%s-%s.txt" (first group) (second group))))
|
||||
(list file-name
|
||||
(-map (lambda (row)
|
||||
(let* ((feed-prefix (alist-get (nth 3 row) arcology-sites nil nil #'equal))
|
||||
(site (nth 3 row))
|
||||
(key (nth 1 row))
|
||||
(feed-url (replace-regexp-in-string (concat site "/") feed-prefix key)))
|
||||
feed-url
|
||||
)
|
||||
)
|
||||
rows))))
|
||||
|
||||
(defun arcology-arroyo-write-feed-list (file lines)
|
||||
(with-temp-buffer
|
||||
(insert (s-join "\n" lines))
|
||||
(write-file file)
|
||||
file))
|
||||
|
||||
(defun arcology-arroyo-make-feed-lists ()
|
||||
(->> (arroyo-db-query [:select * :from arcology-feeds])
|
||||
(-group-by (lambda (row)
|
||||
(list (nth 3 row) (nth 4 row))))
|
||||
(-map (lambda (partitioned)
|
||||
(let* ((partition (first partitioned))
|
||||
(rows (cdr partitioned)))
|
||||
(arcology-arroyo-feeds-to-urls partition rows))))
|
||||
(-map (lambda (group)
|
||||
(apply #'arcology-arroyo-write-feed-list group)))))
|
||||
|
||||
(s-join "\n" (arcology-arroyo-make-feed-lists))
|
||||
#+end_src
|
||||
|
||||
#+results: site-filter
|
||||
~/arroyo-nix/files/feed2toot-uris-garden-private.txt
|
||||
~/arroyo-nix/files/feed2toot-uris-lionsrear-public.txt
|
||||
~/arroyo-nix/files/feed2toot-uris-garden-public.txt
|
||||
~/arroyo-nix/files/feed2toot-uris-arcology-public.txt
|
||||
~/arroyo-nix/files/feed2toot-uris-lionsrear-unlisted.txt
|
||||
|
||||
* INPROGRESS deploy to [[id:20211120T220054.226284][The Wobserver]]
|
||||
:LOGBOOK:
|
||||
- State "INPROGRESS" from "NEXT" [2022-12-28 Wed 12:38]
|
||||
:END:
|
||||
|
||||
=feed2toot= operates with [[https://feed2toot.readthedocs.io/en/latest/configure.html#create-feed2toot-configuration][INI configuration files]] and luckily enough =nixpkgs= includes a way to convert =attrsets= to =ini= files in =lib.generators.toINI=; this is wrapped in a function which takes the basic per-instance configuration details including the uri-list which are generated above and then creates a [[roam:SystemD]] service manifest and timer to poll the site and post the toots. Easy peazy.
|
||||
|
||||
#+ARROYO_NIXOS_MODULE: nixos/feed2toot.nix
|
||||
#+ARROYO_NIXOS_ROLE: server
|
||||
|
||||
#+begin_src nix :tangle ~/arroyo-nix/nixos/feed2toot.nix :noweb yes
|
||||
{ pkgs, lib, config, ... }:
|
||||
|
||||
let
|
||||
mkFeed2TootIni = { instance,
|
||||
uriList,
|
||||
visibility ? "public",
|
||||
credLoc ? config.users.users.feed2toot.home,
|
||||
tagListFile ? pkgs.writeText "empty-taglist" ""}:
|
||||
pkgs.writeText "feed2toot-${instance}"
|
||||
(lib.generators.toINI {}
|
||||
{
|
||||
mastodon = {
|
||||
instance_url = "https://notes.whatthefuck.computer";
|
||||
user_credentials = "${credLoc}/${instance}_feed2toot_usercred.txt";
|
||||
client_credentials = "${credLoc}/${instance}_feed2toot_clientcred.txt";
|
||||
toot_visibility = visibility;
|
||||
};
|
||||
cache.cachefile = "/var/cache/feed2toot/${instance}.db";
|
||||
lock.lock_file = "/run/feed2toot/${instance}.lock";
|
||||
rss.uri_list = builtins.path { path = uriList; };
|
||||
rss.toot = "NEW: {link} \\n {summary}";
|
||||
rss.toot_max_len = 10000;
|
||||
hashtaglist.several_words_hashtags_list = tagListFile;
|
||||
feedparser.accept_bozo_exceptions = true;
|
||||
});
|
||||
|
||||
feeds = [
|
||||
(mkFeed2TootIni {
|
||||
instance = "garden";
|
||||
visibility = "public";
|
||||
uriList = ../files/feed2toot-uris-garden-public.txt;
|
||||
})
|
||||
(mkFeed2TootIni {
|
||||
instance = "lionsrear";
|
||||
visibility = "public";
|
||||
uriList = ../files/feed2toot-uris-lionsrear-public.txt;
|
||||
})
|
||||
(mkFeed2TootIni {
|
||||
instance = "garden";
|
||||
visibility = "unlisted";
|
||||
uriList = ../files/feed2toot-uris-arcology-public.txt;
|
||||
})
|
||||
(mkFeed2TootIni {
|
||||
instance = "lionsrear";
|
||||
visibility = "unlisted";
|
||||
uriList = ../files/feed2toot-uris-lionsrear-unlisted.txt;
|
||||
})
|
||||
(mkFeed2TootIni {
|
||||
instance = "garden";
|
||||
visibility = "private";
|
||||
uriList = ../files/feed2toot-uris-garden-private.txt;
|
||||
})
|
||||
];
|
||||
in {
|
||||
ids.uids.feed2toot = 902;
|
||||
ids.gids.bots = 902;
|
||||
|
||||
users.groups.bots = {
|
||||
gid = config.ids.gids.bots;
|
||||
};
|
||||
users.users."feed2toot" = {
|
||||
home = "/srv/feed2toot/";
|
||||
group = "bots";
|
||||
uid = config.ids.uids.feed2toot;
|
||||
isSystemUser = true;
|
||||
};
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
"d /run/feed2toot 1777 feed2toot bots" # lock file directory
|
||||
"d /var/cache/feed2toot 1777 feed2toot bots" # cache file directory
|
||||
"d /srv/feed2toot 1777 feed2toot bots" # working directory
|
||||
];
|
||||
|
||||
systemd.services.feed2toot = {
|
||||
description = "Feeds to Toots";
|
||||
after = ["pleroma.service"];
|
||||
wantedBy = ["default.target"];
|
||||
script = "${pkgs.feed2toot}/bin/feed2toot" +
|
||||
(lib.concatMapStrings (feed: " -c " + feed) feeds);
|
||||
# "-c ${gardenIni} -c ${lionsrearIni} -c ${engineIni}" ;
|
||||
serviceConfig.User = "feed2toot";
|
||||
serviceConfig.WorkingDirectory = "/srv/feed2toot";
|
||||
startAt = "30min";
|
||||
};
|
||||
}
|
||||
#+end_src
|
||||
|
||||
* NEXT remove bs4 html filtering
|
||||
|
||||
pleroma supports embedding HTML so we should use that.
|
||||
Loading…
Reference in New Issue