rrix
/
feediverse
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

do all the multi-site stuff in a single invocation

master
Ryan Rix 2023-01-25 14:03:26 -10:00
parent 361dd98c64
commit ce40985b7c
2 changed files with 65 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
config.yml.sample Normal file
View File

@ -0,0 +1,15 @@
tokens:
lionsrear: *lionsrear-creds
garden: *garden-creds
cce: *garden-creds
arcology: *garden-creds
feeds_index: https://thelionsrear.com/feeds.json
post_template: >-
NEW by @rrix@notes.whatthefuck.computer:
{title}
{summary}
{url} {hashtags}
updated: '2023-01-25T06:13:50.343361+00:00'
url: https://notes.whatthefuck.computer

120
feediverse.py
View File

@ -1,4 +1,5 @@
#!/usr/bin/env python3
# Make sure to edit this in cce/feediverse.org !!!
import os
import re
@ -26,9 +27,6 @@ def main():
parser.add_argument("-c", "--config",
help="config file to use",
default=os.path.expanduser(DEFAULT_CONFIG_FILE))
parser.add_argument("-f", "--feeds",
help="URL to fetch feed list from",
default="https://thelionsrear.com/feeds.json")
args = parser.parse_args()
config_file = args.config
@ -37,46 +35,45 @@ def main():
print("using config file", config_file)
if not os.path.isfile(config_file):
setup(config_file)
setup(config_file)
config = read_config(config_file)
masto = Mastodon(
api_base_url=config['url'],
client_id=config['client_id'],
client_secret=config['client_secret'],
access_token=config['access_token']
)
config['feeds'] = fetch_dynamic_feeds(config['name'], args.feeds)
newest_post = config['updated']
for feed in config['feeds']:
if args.verbose:
print(f"fetching {feed['url']} entries since {config['updated']}")
for entry in get_feed(feed['url'], config['updated']):
newest_post = max(newest_post, entry['updated'])
per_site_feeds = fetch_dynamic_feeds(config['feeds_index'])
for site, feeds in per_site_feeds.items():
masto = Mastodon(
api_base_url=config['url'],
client_id=config['tokens'][site]['client_id'],
client_secret=config['tokens'][site]['client_secret'],
access_token=config['tokens'][site]['access_token']
)
for feed in feeds:
if args.verbose:
print(entry)
if args.dry_run:
print("trial run, not tooting ", entry["title"][:50])
continue
masto.status_post(feed['template'].format(**entry)[:499],
content_type='text/html',
visbility=feed['visibility'])
print(f"fetching {feed['url']} entries since {config['updated']}")
for entry in get_feed(feed['url'], config['updated']):
newest_post = max(newest_post, entry['updated'])
if args.verbose:
print(entry)
if args.dry_run:
print("trial run, not tooting ", entry["title"][:50])
continue
masto.status_post(feed['template'].format(**entry),
content_type='text/html',
visbility=feed['visibility'])
if not args.dry_run:
config['updated'] = newest_post.isoformat()
save_config(config, config_file)
def fetch_dynamic_feeds(site_name, feeds_url):
def fetch_dynamic_feeds(feeds_url):
feeds = requests.get(feeds_url).json()
return [
dict(url=x['url'],
visibility=x['visibility'],
template='NEW by @rrix@notes.whatthefuck.computer: {url} {title}\n{summary}')
for x in feeds if x['site'] == site_name
]
feeds_by_site = dict()
for feed in feeds:
feeds_by_site[feed['site']] = feeds_by_site.get(feed['site'], []) + [feed]
return feeds_by_site
def get_feed(feed_url, last_update):
feed = feedparser.parse(feed_url)
@ -96,8 +93,6 @@ def get_entry(entry):
hashtags.append('#{}'.format(t))
summary = entry.get('summary', '')
content = entry.get('content', '') or ''
# if content:
# content = cleanup(content[0].get('value', ''))
url = entry.id
return {
'url': url,
@ -119,24 +114,6 @@ def cleanup(text):
text = re.sub('\n\n\n+', '\n\n', text, flags=re.M)
return text.strip()
def find_urls(html):
if not html:
return
urls = []
soup = BeautifulSoup(html, 'html.parser')
for tag in soup.find_all(["a", "img"]):
if tag.name == "a":
url = tag.get("href")
elif tag.name == "img":
url = tag.get("src")
if url and url not in urls:
urls.append(url)
return urls
def yes_no(question):
res = input(question + ' [y/n] ')
return res.lower() in "y1"
def save_config(config, config_file):
copy = dict(config)
with open(config_file, 'w') as fh:
@ -153,17 +130,18 @@ def read_config(config_file):
config.update(cfg)
return config
def yes_no(question):
res = input(question + ' [y/n] ')
return res.lower() in "y1"
def setup(config_file):
url = input('What is your Mastodon Instance URL? ')
have_app = yes_no('Do you have your app credentials already?')
if have_app:
name = 'feediverse'
client_id = input('What is your app\'s client id: ')
client_secret = input('What is your client secret: ')
access_token = input('access_token: ')
else:
print("Ok, I'll need a few things in order to get your access token")
name = input('app name (e.g. feediverse): ')
url = input('What is your Fediverse Instance URL? ')
feeds_index = input("What is the arcology feed index URL? ")
tokens = dict()
for site in fetch_dynamic_feeds(feeds_index).keys():
print(f"Configuring for {site}...")
print("I'll need a few things in order to get your access token")
name = input('app name (e.g. feediverse): ') or "feediverse"
client_id, client_secret = Mastodon.create_app(
api_base_url=url,
client_name=name,
@ -175,17 +153,19 @@ def setup(config_file):
m = Mastodon(client_id=client_id, client_secret=client_secret, api_base_url=url)
access_token = m.log_in(username, password)
feed_url = input('RSS/Atom feed URL to watch: ')
tokens[site] = {
'client_id': client_id,
'client_secret': client_secret,
'access_token': access_token,
}
old_posts = yes_no('Shall already existing entries be tooted, too?')
config = {
'name': name,
'url': url,
'client_id': client_id,
'client_secret': client_secret,
'access_token': access_token,
'feeds': [
{'url': feed_url, 'template': '{title} {url}'}
]
'feeds_index': feeds_index,
'tokens': tokens,
'template': '{title} {summary} {url}'
}
if not old_posts:
config['updated'] = datetime.now(tz=timezone.utc).isoformat()