add feed exporting using django templates and metadata extractor

arcology.org

@@ -12,6 +12,7 @@ from __future__ import annotations
 from typing import Optional, List
 from django.db import models
 from django.conf import settings
+import arrow
 
 import arroyo.arroyo_rs as native
 
@@ -153,8 +154,11 @@ class Page(models.Model):
         ]
 
     def collect_links(self):
+        my_headings = self.file.heading_set.all()
         link_objs = self.file.outbound_links.all()
-        ret = dict()
+        ret = {
+          h.node_id: h.to_url() for h in my_headings
+        }
         for el in link_objs:
             try:
                 h = el.dest_heading
@@ -172,9 +176,12 @@ class Page(models.Model):
         return set(roam.models.Link.objects.filter(dest_heading__in=my_headings))
 
 
-    def to_html(self, links):
+    def to_html(self, links, headings=[], include_subheadings=False):
         opts = native.ExportOptions(
-            link_retargets=links
+            link_retargets=links,
+            limit_headings=headings,
+            include_subheadings=include_subheadings,
+            ignore_tags=[],
         )
         return native.htmlize_file(self.file.path, opts)
 
@@ -262,6 +269,7 @@ class Feed(models.Model):
     visibility = models.CharField(max_length=512, choices=POST_VISIBILITY)
 
     def to_atom(self, links):
+        raise "Re-implement this!"
         opts = native.ExportOptions(
             link_retargets=links
         )
@@ -334,6 +342,99 @@ migrations.CreateModel(
     ],
 ),
 #+end_src
+** NEXT FeedEntry
+
+A FeedEntry is a Heading with a PUBDATE property that exists on a page w/ ARCOLOGY_FEED Keyword
+
+#+begin_src python
+feed_kws = roam.models.Keyword.objects.filter(value="garden/updates.xml", keyword="ARCOLOGY_FEED")
+
+headings = [
+    item
+    for kw in feed_kws
+    for item in kw.path.heading_set \
+      .filter(headingproperty__keyword="PUBDATE") \
+      .exclude(tag__tag__in=["noexport", "NOEXPORT"]) \
+      .all()
+]
+
+[
+    h.title for h in headings
+]
+#+end_src
+
+#+begin_src python :tangle arcology/models.py
+class FeedEntry(models.Model):
+    POST_VISIBILITY = [
+        ("unlisted", "Unlisted"),
+        ("private", "Private"),
+        ("public", "Public"),
+        ("direct", "direct"),  # might be different, XXX
+    ]
+
+    heading = models.ForeignKey(
+        roam.models.Heading,
+        on_delete=models.CASCADE,
+    )
+    feed = models.ForeignKey(
+        Feed,
+        on_delete=models.CASCADE,
+    )
+    route_key = models.CharField(max_length=512)
+    site = models.ForeignKey(
+        Site,
+        on_delete=models.CASCADE,
+    )
+    title = models.CharField(max_length=512)
+    visibility = models.CharField(max_length=512, choices=POST_VISIBILITY)
+    pubdate = models.DateTimeField(auto_now=False)
+
+    def to_html(self, links):
+        opts = native.ExportOptions(
+            link_retargets=links,
+            limit_headings=[self.heading.node_id],
+            include_subheadings=True,
+            ignore_tags=[],
+        )
+        return native.htmlize_file(self.heading.path.path, opts)
+
+    @classmethod
+    def create_from_arroyo(cls, doc: native.Document) -> List[Feed] | None:
+        route_key = next(iter(doc.collect_keywords("ARCOLOGY_FEED")), None)
+        if not route_key:
+            return None
+        visibility = next(
+            iter(doc.collect_keywords("ARCOLOGY_TOOT_VISIBILITY")), "private"
+        )
+        site = Site.from_route(route_key)
+        # f = roam.models.File.objects.get(path=doc.path)
+        feed = Feed.objects.get(route_key=route_key)
+
+        rets = []
+        for nheading in doc.headings:
+          if nheading.id is not None:
+            heading = roam.models.Heading.objects.get(node_id=nheading.id)
+            pdqs = heading.headingproperty_set.filter(keyword="PUBDATE")
+            if not pdqs.exists():
+                continue
+            v = pdqs.first().value
+            pubdate = arrow.get(v, "YYYY-MM-DD ddd H:mm").format(arrow.FORMAT_RFC3339)
+            title = heading.title
+            rets += [cls.objects.get_or_create(
+                heading=heading,
+                feed=feed,
+                route_key=route_key,
+                title=title,
+                pubdate=pubdate,
+                visibility=visibility,
+                site=site,
+            )[0]]
+        # root_heading = f.heading_set.filter(level=0)[0]
+        # title = root_heading.title
+
+        return rets
+#+end_src
+
 ** Database Migrations
 
 #+begin_src python :tangle arcology/migrations/__init__.py
@@ -392,6 +493,10 @@ class PageAdmin(admin.ModelAdmin):
 @admin.register(arcology.models.Feed)
 class FeedAdmin(admin.ModelAdmin):
     pass
+
+@admin.register(arcology.models.FeedEntry)
+class FeedEntryAdmin(admin.ModelAdmin):
+    list_display = ["heading", "route_key", "pubdate", "title"]
 #+end_src
 
 * NEXT the web server
@@ -454,6 +559,9 @@ def index(request):
 #+end_src
 
 ** org-page
+:PROPERTIES:
+:ID:       20240202T144002.656093
+:END:
 :LOGBOOK:
 - State "INPROGRESS" from              [2023-12-20 Wed 17:48]
 :END:
@@ -634,6 +742,9 @@ all the pandoc based feed generator stuff will need to be recreated or
 bodged in, and all that probably should go in its own django app.
 
 #+begin_src python :tangle arcology/views.py
+import arrow 
+import roam.models
+
 def feed(request, key):
     site = Site.from_request(request)
     if site.key == "localhost":
@@ -646,9 +757,62 @@ def feed(request, key):
 
     the_feed = get_object_or_404(Feed, route_key=full_key)
     links = the_feed.file.page_set.first().collect_links()
-    feed_xml = the_feed.to_atom(links)
+    entries = the_feed.feedentry_set.order_by("-pubdate").all()[:10]
+    html_map = {
+        entry.heading.node_id: entry.to_html(links=links) for entry in entries
+    }
+    pubdate_map = {
+        entry.heading.node_id: arrow.get(entry.pubdate).format(arrow.FORMAT_RFC3339) for entry in entries
+    }
 
-    return HttpResponse(feed_xml,content_type="application/atom+xml")
+    page_author = roam.models.Keyword.objects.get(keyword="AUTHOR", path=the_feed.file).value
+
+    # return HttpResponse("",content_type="application/atom+xml")
+    return render(request, "arcology/feed.xml", dict(
+        title="Test",
+        page_url=the_feed.file.page_set.first().urlize_self(),
+        author=page_author,
+        updated_at=arrow.get(entries[0].pubdate).format(arrow.FORMAT_RFC3339),
+
+        feed_entries=entries,
+        htmls=html_map,
+        pubdates=pubdate_map,
+
+        links=links,
+    ), content_type="application/atom+xml")
+
+from django.template.defaulttags import register
+
+@register.filter
+def get_item(dictionary, key):
+    return dictionary.get(key)
+#+end_src
+
+Feed template:
+
+#+begin_src jinja2 :tangle arcology/templates/arcology/feed.xml :mkdirp yes
+<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+
+  <title>{{ title }}</title>
+  <link href="{{ page_url }}"/>
+  <updated>{{ updated_at }}</updated>
+  <author>
+    <name>{{ author }}</name>
+  </author>
+  <id>{{ page_url }}</id>
+
+{% for entry in feed_entries %}
+  <entry>
+    <title>{{ entry.title }}</title>
+    <link href="{{ links | get_item:entry.heading.node_id }}"/>
+    <id>urn:uid:{{ entry.heading.node_id }}</id>
+    <updated>{{ pubdates | get_item:entry.heading.node_id }}</updated>
+    <summary type="html">{{ htmls | get_item:entry.heading.node_id }}</summary>
+  </entry>
+{% endfor %}
+
+</feed>
 #+end_src
 
 ** Per-Site link color CSS endpoint

arcology/admin.py

@@ -20,4 +20,8 @@ class PageAdmin(admin.ModelAdmin):
 @admin.register(arcology.models.Feed)
 class FeedAdmin(admin.ModelAdmin):
     pass
+
+@admin.register(arcology.models.FeedEntry)
+class FeedEntryAdmin(admin.ModelAdmin):
+    list_display = ["heading", "route_key", "pubdate", "title"]
 # admin:1 ends here

arcology/migrations/0002_alter_sitedomain_domain.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.6 on 2024-02-05 03:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("arcology", "0001_base"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="sitedomain",
+            name="domain",
+            field=models.CharField(max_length=512),
+        ),
+    ]

arcology/migrations/0003_feedentry.py

@@ -0,0 +1,63 @@
+# Generated by Django 4.2.6 on 2024-02-05 03:41
+
+from django.db import migrations, models
+import django.db.models.deletion
+import datetime
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("roam", "0002_alter_link_source_file_headingproperty"),
+        ("arcology", "0002_alter_sitedomain_domain"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="FeedEntry",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("route_key", models.CharField(max_length=512)),
+                ("title", models.CharField(max_length=512)),
+                (
+                    "visibility",
+                    models.CharField(
+                        choices=[
+                            ("unlisted", "Unlisted"),
+                            ("private", "Private"),
+                            ("public", "Public"),
+                            ("direct", "direct"),
+                        ],
+                        max_length=512,
+                    ),
+                ),
+                (
+                    "heading",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="roam.heading"
+                    ),
+                ),
+                (
+                    "site",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="arcology.site"
+                    ),
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name="feedentry",
+            name="pubdate",
+            field=models.DateTimeField(
+                default=datetime.datetime(2024, 2, 5, 4, 9, 19, 212818)
+            ),
+            preserve_default=False,
+        ),
+    ]

arcology/migrations/0004_feedentry_feed.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.6 on 2024-02-05 04:46
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("arcology", "0003_feedentry"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="feedentry",
+            name="feed",
+            field=models.ForeignKey(
+                default=None,
+                on_delete=django.db.models.deletion.CASCADE,
+                to="arcology.feed",
+            ),
+            preserve_default=False,
+        ),
+    ]

arcology/models.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from typing import Optional, List
 from django.db import models
 from django.conf import settings
+import arrow
 
 import arroyo.arroyo_rs as native
 
@@ -87,8 +88,11 @@ class Page(models.Model):
         ]
 
     def collect_links(self):
+        my_headings = self.file.heading_set.all()
         link_objs = self.file.outbound_links.all()
-        ret = dict()
+        ret = {
+          h.node_id: h.to_url() for h in my_headings
+        }
         for el in link_objs:
             try:
                 h = el.dest_heading
@@ -106,9 +110,12 @@ class Page(models.Model):
         return set(roam.models.Link.objects.filter(dest_heading__in=my_headings))
 
 
-    def to_html(self, links):
+    def to_html(self, links, headings=[], include_subheadings=False):
         opts = native.ExportOptions(
-            link_retargets=links
+            link_retargets=links,
+            limit_headings=headings,
+            include_subheadings=include_subheadings,
+            ignore_tags=[],
         )
         return native.htmlize_file(self.file.path, opts)
 
@@ -155,6 +162,7 @@ class Feed(models.Model):
     visibility = models.CharField(max_length=512, choices=POST_VISIBILITY)
 
     def to_atom(self, links):
+        raise "Re-implement this!"
         opts = native.ExportOptions(
             link_retargets=links
         )
@@ -187,3 +195,75 @@ class Feed(models.Model):
             **kwargs
         )
 # Feed:1 ends here
+
+# [[file:../arcology.org::*FeedEntry][FeedEntry:2]]
+class FeedEntry(models.Model):
+    POST_VISIBILITY = [
+        ("unlisted", "Unlisted"),
+        ("private", "Private"),
+        ("public", "Public"),
+        ("direct", "direct"),  # might be different, XXX
+    ]
+
+    heading = models.ForeignKey(
+        roam.models.Heading,
+        on_delete=models.CASCADE,
+    )
+    feed = models.ForeignKey(
+        Feed,
+        on_delete=models.CASCADE,
+    )
+    route_key = models.CharField(max_length=512)
+    site = models.ForeignKey(
+        Site,
+        on_delete=models.CASCADE,
+    )
+    title = models.CharField(max_length=512)
+    visibility = models.CharField(max_length=512, choices=POST_VISIBILITY)
+    pubdate = models.DateTimeField(auto_now=False)
+
+    def to_html(self, links):
+        opts = native.ExportOptions(
+            link_retargets=links,
+            limit_headings=[self.heading.node_id],
+            include_subheadings=True,
+            ignore_tags=[],
+        )
+        return native.htmlize_file(self.heading.path.path, opts)
+
+    @classmethod
+    def create_from_arroyo(cls, doc: native.Document) -> List[Feed] | None:
+        route_key = next(iter(doc.collect_keywords("ARCOLOGY_FEED")), None)
+        if not route_key:
+            return None
+        visibility = next(
+            iter(doc.collect_keywords("ARCOLOGY_TOOT_VISIBILITY")), "private"
+        )
+        site = Site.from_route(route_key)
+        # f = roam.models.File.objects.get(path=doc.path)
+        feed = Feed.objects.get(route_key=route_key)
+
+        rets = []
+        for nheading in doc.headings:
+          if nheading.id is not None:
+            heading = roam.models.Heading.objects.get(node_id=nheading.id)
+            pdqs = heading.headingproperty_set.filter(keyword="PUBDATE")
+            if not pdqs.exists():
+                continue
+            v = pdqs.first().value
+            pubdate = arrow.get(v, "YYYY-MM-DD ddd H:mm").format(arrow.FORMAT_RFC3339)
+            title = heading.title
+            rets += [cls.objects.get_or_create(
+                heading=heading,
+                feed=feed,
+                route_key=route_key,
+                title=title,
+                pubdate=pubdate,
+                visibility=visibility,
+                site=site,
+            )[0]]
+        # root_heading = f.heading_set.filter(level=0)[0]
+        # title = root_heading.title
+
+        return rets
+# FeedEntry:2 ends here

arcology/settings/generators.py

@@ -11,9 +11,11 @@ ARCOLOGY_EXTRACTORS = [
   "roam.models.Heading", # [[id:arcology/django/roam][Arcology Roam Models]]'s heading entity
   "roam.models.Reference", # [[id:arcology/django/roam][Arcology Roam Models]]'s [[id:cce/org-roam][org-roam]] reference entity
   "roam.models.Tag", # [[id:arcology/django/roam][Arcology Roam Models]]'s heading tag entity
+  "roam.models.HeadingProperty", # [[id:arcology/django/roam][Arcology Roam Models]]'s heading properties map
   "roam.models.Link", # [[id:arcology/django/roam][Arcology Roam Models]] page-to-page links
   "arcology.models.Page", # [[id:arcology/django/arcology-models][The Arcology's Data Models]]'s Page entity
   "arcology.models.Feed", # [[id:arcology/django/arcology-models][The Arcology's Data Models]]'s Feed entity, probably move to its own module some day...
+  "arcology.models.FeedEntry", # 
   
 ]
 
@@ -52,6 +54,7 @@ ROAM_ALLOWED_KEYWORDS = [
   "ARROYO_SYSTEM_ROLE", # Constrain the Nix expressions on the page to only load in this role (can be specified repeatedly)
   "ARROYO_SYSTEM_EXCLUDE", # Prevent the Nix expressions on the page from loading in the listed role (can be specified repeatedly)
   "ARROYO_DIRENV_DIR", # Load [[id:45fc2a02-fcd0-40c6-a29e-897c0ee7b1c7][direnv]] from a different directory than the loaded buffer's directory. Useful for [[id:cce/literate_programming][Org Babel]].
+  "AUTHOR", # Used to populate the author in RSS feeds and page metadata
   
   "BIRTHDAY", # Used in [[id:8803fda8-20cd-4d62-878d-dcb1b7853183][People]] pages.
   "LOCATION", # Used in [[id:8803fda8-20cd-4d62-878d-dcb1b7853183][People]] pages.

arcology/templates/arcology/feed.xml

@@ -0,0 +1,24 @@
+{# [[file:../../../arcology.org::*feed][feed:2]] #}
+<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+
+  <title>{{ title }}</title>
+  <link href="{{ page_url }}"/>
+  <updated>{{ updated_at }}</updated>
+  <author>
+    <name>{{ author }}</name>
+  </author>
+  <id>{{ page_url }}</id>
+
+{% for entry in feed_entries %}
+  <entry>
+    <title>{{ entry.title }}</title>
+    <link href="{{ links | get_item:entry.heading.node_id }}"/>
+    <id>urn:uid:{{ entry.heading.node_id }}</id>
+    <updated>{{ pubdates | get_item:entry.heading.node_id }}</updated>
+    <summary type="html">{{ htmls | get_item:entry.heading.node_id }}</summary>
+  </entry>
+{% endfor %}
+
+</feed>
+{# feed:2 ends here #}

arcology/views.py

@@ -54,6 +54,9 @@ def sitemap(request):
 # sitemap:1 ends here
 
 # [[file:../arcology.org::*feed][feed:1]]
+import arrow 
+import roam.models
+
 def feed(request, key):
     site = Site.from_request(request)
     if site.key == "localhost":
@@ -66,9 +69,35 @@ def feed(request, key):
 
     the_feed = get_object_or_404(Feed, route_key=full_key)
     links = the_feed.file.page_set.first().collect_links()
-    feed_xml = the_feed.to_atom(links)
+    entries = the_feed.feedentry_set.order_by("-pubdate").all()[:10]
+    html_map = {
+        entry.heading.node_id: entry.to_html(links=links) for entry in entries
+    }
+    pubdate_map = {
+        entry.heading.node_id: arrow.get(entry.pubdate).format(arrow.FORMAT_RFC3339) for entry in entries
+    }
 
-    return HttpResponse(feed_xml,content_type="application/atom+xml")
+    page_author = roam.models.Keyword.objects.get(keyword="AUTHOR", path=the_feed.file).value
+
+    # return HttpResponse("",content_type="application/atom+xml")
+    return render(request, "arcology/feed.xml", dict(
+        title="Test",
+        page_url=the_feed.file.page_set.first().urlize_self(),
+        author=page_author,
+        updated_at=arrow.get(entries[0].pubdate).format(arrow.FORMAT_RFC3339),
+
+        feed_entries=entries,
+        htmls=html_map,
+        pubdates=pubdate_map,
+
+        links=links,
+    ), content_type="application/atom+xml")
+
+from django.template.defaulttags import register
+
+@register.filter
+def get_item(dictionary, key):
+    return dictionary.get(key)
 # feed:1 ends here
 
 # [[file:../arcology.org::*Per-Site link color CSS endpoint][Per-Site link color CSS endpoint:1]]

configuration.org

@@ -271,12 +271,14 @@ It should be feasible to add new data types and models and generators to the Arc
 These models are only persisted if the page is going to be published and are used by the core [[id:arcology/django/arcology-models][The Arcology Web Server]] to set up the page and render backlinks and tag pages and whatnot.
 
 #+name: arcology_extractors
-| roam.models.Heading   | [[id:arcology/django/roam][Arcology Roam Models]]'s heading entity                                                 |
-| roam.models.Reference | [[id:arcology/django/roam][Arcology Roam Models]]'s [[id:cce/org-roam][org-roam]] reference entity                                      |
-| roam.models.Tag       | [[id:arcology/django/roam][Arcology Roam Models]]'s heading tag entity                                             |
-| roam.models.Link      | [[id:arcology/django/roam][Arcology Roam Models]] page-to-page links                                               |
-| arcology.models.Page  | [[id:arcology/django/arcology-models][The Arcology's Data Models]]'s Page entity                                              |
-| arcology.models.Feed  | [[id:arcology/django/arcology-models][The Arcology's Data Models]]'s Feed entity, probably move to its own module some day... |
+| roam.models.Heading         | [[id:arcology/django/roam][Arcology Roam Models]]'s heading entity                                                 |
+| roam.models.Reference       | [[id:arcology/django/roam][Arcology Roam Models]]'s [[id:cce/org-roam][org-roam]] reference entity                                      |
+| roam.models.Tag             | [[id:arcology/django/roam][Arcology Roam Models]]'s heading tag entity                                             |
+| roam.models.HeadingProperty | [[id:arcology/django/roam][Arcology Roam Models]]'s heading properties map                                         |
+| roam.models.Link            | [[id:arcology/django/roam][Arcology Roam Models]] page-to-page links                                               |
+| arcology.models.Page        | [[id:arcology/django/arcology-models][The Arcology's Data Models]]'s Page entity                                              |
+| arcology.models.Feed        | [[id:arcology/django/arcology-models][The Arcology's Data Models]]'s Feed entity, probably move to its own module some day... |
+| arcology.models.FeedEntry   |                                                                                       |
 
 The [[id:arroyo/django/generators][Arroyo Generators]] are run regardless of whether a file is published. These are a dictionary so that the user can specify a module when invoking the [[id:20231217T154938.132553][Arcology =generate= Command]].
 
@@ -332,6 +334,7 @@ These are used to provide system features:
 | ARROYO_SYSTEM_ROLE       | Constrain the Nix expressions on the page to only load in this role (can be specified repeatedly)     |
 | ARROYO_SYSTEM_EXCLUDE    | Prevent the Nix expressions on the page from loading in the listed role (can be specified repeatedly) |
 | ARROYO_DIRENV_DIR        | Load [[id:45fc2a02-fcd0-40c6-a29e-897c0ee7b1c7][direnv]] from a different directory than the loaded buffer's directory. Useful for [[id:cce/literate_programming][Org Babel]].      |
+| AUTHOR                   | Used to populate the author in RSS feeds and page metadata                                            |
 
 These are used in some of my custom reports:
 #+NAME: my_roam_allowed_keywords

default.nix

@@ -22,6 +22,7 @@ python3.pkgs.buildPythonPackage rec {
     setuptools
     django
     polling
+    arrow
   ]);
 
   meta = with lib; {

flake.lock

@@ -6,15 +6,16 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1705480217,
-        "narHash": "sha256-1Xx88iGThODMczFLVj06CuhfFfL8cXPbGrlg0gV7Ees=",
-        "ref": "refs/heads/main",
-        "rev": "aacd62b4f67144634a0bb6b14e72fae690380348",
-        "revCount": 138,
+        "lastModified": 1707101614,
+        "narHash": "sha256-ZZbBX1/B0eku2Ww2luX4MzSIP5gONa6Tf2mmkmdQ2Ss=",
+        "ref": "rr/subheading-export",
+        "rev": "52258e2db7047335a9ee7b51121960b394fb3619",
+        "revCount": 146,
         "type": "git",
         "url": "https://code.rix.si/rrix/arroyo"
       },
       "original": {
+        "ref": "rr/subheading-export",
         "type": "git",
         "url": "https://code.rix.si/rrix/arroyo"
       }

flake.nix

@@ -4,7 +4,7 @@
 
   inputs.nixpkgs.follows = "arroyo_rs/nixpkgs";
   inputs.flake-utils.url = "github:numtide/flake-utils";
-  inputs.arroyo_rs.url = "git+https://code.rix.si/rrix/arroyo";
+  inputs.arroyo_rs.url = "git+https://code.rix.si/rrix/arroyo?ref=rr/subheading-export";
 
   outputs = { self, nixpkgs, flake-utils, arroyo_rs }:
     flake-utils.lib.eachDefaultSystem (system:

interfaces.org

@@ -223,9 +223,10 @@ These will be the "public" interfaces for the generators, a set of Emacs functio
          (output (shell-command-to-string cmd)))
     output))
 
-(defun arroyo-generate-imports (module &optional role destination)
+(defun arroyo-generate-imports (module &optional role destination  do-sort)
   (setenv "ARCOLOGY_DB_PATH" "/home/rrix/org/arcology-django/db.sqlite3")
   (let* ((flake-path "path:/home/rrix/org/arcology-django")
+         (do-sort (and do-sort t))
          (cmd (s-join " " (list (format "nix run %s#arcology" flake-path)
                                 "--"
                                 "generate -m"
@@ -235,7 +236,8 @@ These will be the "public" interfaces for the generators, a set of Emacs functio
                                 (when destination
                                   (format "-d %s" destination))
                                 "2>/dev/null"
-                                " | sort")))
+                                (when do-sort
+                                  " | sort"))))
          (output (shell-command-to-string cmd)))
     output))
 

pyproject.toml

@@ -5,7 +5,7 @@ version = "0.0.1"
 description = "org-mode metadata query engine, publishing platform, and computer metaprogrammer"
 # license = "Hey Smell This"
 readme = "README.md"
-dependencies = ["click ~=8.1", "django ~= 4.2", "django-stub", "polling", "arroyo"]
+dependencies = ["click ~=8.1", "django ~= 4.2", "django-stub", "polling", "arroyo", "arrow"]
 requires-python = ">=3.10"
 authors = [
     { name = "Ryan Rix", email = "code@whatthefuck.computer" }

roam.org

@@ -141,6 +141,31 @@ class Heading(models.Model):
         ]
 #+END_SRC
 
+** Properties
+
+#+begin_src python :tangle roam/models.py
+class HeadingProperty(models.Model):
+    heading = models.ForeignKey(
+        Heading,
+        on_delete=models.CASCADE,
+        db_column="node_id",
+    )
+    keyword = models.CharField(max_length=256)
+    value = models.CharField(max_length=256)
+
+    @classmethod
+    def create_from_arroyo(cls, doc: native.Document) -> List[Tag]:
+        return [
+            cls.objects.get_or_create(
+                heading=Heading.objects.get(node_id=heading.id),
+                keyword=key, value=value
+            )[0]
+            for heading in doc.headings or []
+            for key, value in (heading.properties or {}).items()
+            if heading.id is not None
+        ]
+#+end_src
+
 ** Tag
 
 #+BEGIN_SRC python :tangle roam/models.py
@@ -596,6 +621,9 @@ class KeywordInline(admin.TabularInline):
 class HeadingInline(admin.TabularInline):
     model = roam.models.Heading
 
+class PropertyInline(admin.TabularInline):
+    model = roam.models.HeadingProperty
+
 class TagInline(admin.TabularInline):
     model = roam.models.Tag
 
@@ -603,6 +631,11 @@ class ReferenceInline(admin.TabularInline):
     model = roam.models.Reference
 
 
+@admin.register(roam.models.HeadingProperty)
+class PropertyAdmin(admin.ModelAdmin):
+    list_display = ["heading", "keyword", "value"]
+
+
 @admin.register(roam.models.Keyword)
 class KeywordAdmin(admin.ModelAdmin):
     list_display = ["path", "keyword", "value"]
@@ -627,7 +660,8 @@ class HeadingAdmin(admin.ModelAdmin):
     list_display = ["node_id", "path"]
     inlines = [
         TagInline,
-        ReferenceInline
+        ReferenceInline,
+        PropertyInline,
     ]
 
 admin.site.register(roam.models.Link)
@@ -653,4 +687,3 @@ class RoamConfig(AppConfig):
     default_auto_field = "django.db.models.BigAutoField"
     name = "roam"
 #+end_src
-

roam/admin.py

@@ -10,6 +10,9 @@ class KeywordInline(admin.TabularInline):
 class HeadingInline(admin.TabularInline):
     model = roam.models.Heading
 
+class PropertyInline(admin.TabularInline):
+    model = roam.models.HeadingProperty
+
 class TagInline(admin.TabularInline):
     model = roam.models.Tag
 
@@ -17,6 +20,11 @@ class ReferenceInline(admin.TabularInline):
     model = roam.models.Reference
 
 
+@admin.register(roam.models.HeadingProperty)
+class PropertyAdmin(admin.ModelAdmin):
+    list_display = ["heading", "keyword", "value"]
+
+
 @admin.register(roam.models.Keyword)
 class KeywordAdmin(admin.ModelAdmin):
     list_display = ["path", "keyword", "value"]
@@ -41,7 +49,8 @@ class HeadingAdmin(admin.ModelAdmin):
     list_display = ["node_id", "path"]
     inlines = [
         TagInline,
-        ReferenceInline
+        ReferenceInline,
+        PropertyInline,
     ]
 
 admin.site.register(roam.models.Link)

roam/migrations/0002_alter_link_source_file_headingproperty.py

@@ -0,0 +1,46 @@
+# Generated by Django 4.2.6 on 2024-02-05 03:04
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("roam", "0001_base"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="link",
+            name="source_file",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="outbound_links",
+                to="roam.file",
+            ),
+        ),
+        migrations.CreateModel(
+            name="HeadingProperty",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("keyword", models.CharField(max_length=256)),
+                ("value", models.CharField(max_length=256)),
+                (
+                    "heading",
+                    models.ForeignKey(
+                        db_column="node_id",
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="roam.heading",
+                    ),
+                ),
+            ],
+        ),
+    ]

roam/models.py

@@ -121,6 +121,29 @@ class Heading(models.Model):
         ]
 # Heading:1 ends here
 
+# [[file:../roam.org::*Properties][Properties:1]]
+class HeadingProperty(models.Model):
+    heading = models.ForeignKey(
+        Heading,
+        on_delete=models.CASCADE,
+        db_column="node_id",
+    )
+    keyword = models.CharField(max_length=256)
+    value = models.CharField(max_length=256)
+
+    @classmethod
+    def create_from_arroyo(cls, doc: native.Document) -> List[Tag]:
+        return [
+            cls.objects.get_or_create(
+                heading=Heading.objects.get(node_id=heading.id),
+                keyword=key, value=value
+            )[0]
+            for heading in doc.headings or []
+            for key, value in (heading.properties or {}).items()
+            if heading.id is not None
+        ]
+# Properties:1 ends here
+
 # [[file:../roam.org::*Tag][Tag:1]]
 class Tag(models.Model):
     class Meta:

scaffolding.org

@@ -17,7 +17,7 @@ version = "0.0.1"
 description = "org-mode metadata query engine, publishing platform, and computer metaprogrammer"
 # license = "Hey Smell This"
 readme = "README.md"
-dependencies = ["click ~=8.1", "django ~= 4.2", "django-stub", "polling", "arroyo"]
+dependencies = ["click ~=8.1", "django ~= 4.2", "django-stub", "polling", "arroyo", "arrow"]
 requires-python = ">=3.10"
 authors = [
     { name = "Ryan Rix", email = "code@whatthefuck.computer" }
@@ -65,6 +65,7 @@ python3.pkgs.buildPythonPackage rec {
     setuptools
     django
     polling
+    arrow
   ]);
 
   meta = with lib; {
@@ -86,7 +87,7 @@ Nix is really going this direction, I'm not sure it's worthwhile but I'm going t
 
   inputs.nixpkgs.follows = "arroyo_rs/nixpkgs";
   inputs.flake-utils.url = "github:numtide/flake-utils";
-  inputs.arroyo_rs.url = "git+https://code.rix.si/rrix/arroyo";
+  inputs.arroyo_rs.url = "git+https://code.rix.si/rrix/arroyo?ref=rr/subheading-export";
 
   outputs = { self, nixpkgs, flake-utils, arroyo_rs }:
     flake-utils.lib.eachDefaultSystem (system:
@@ -146,6 +147,7 @@ let
     django-stubs.override { django = django_4; }
     django-stubs-ext.override { django = django_4; }
     polling
+    arrow
   ]);
 in pkgs.mkShell {
   packages = (with pkgs; [

shell.nix

@@ -15,6 +15,7 @@ let
     django-stubs.override { django = django_4; }
     django-stubs-ext.override { django = django_4; }
     polling
+    arrow
   ]);
 in pkgs.mkShell {
   packages = (with pkgs; [