fix sync command

arcology.org

@@ -3,7 +3,9 @@
 :END:
 #+TITLE: The Arcology's Data Models and Web Server
 #+filetags: :Project:Arcology:
+
 #+ARCOLOGY_KEY: arcology/django/arcology
+#+ARCOLOGY_ALLOW_CRAWL: t
 
 * Data Models for Sites, Web Features, and Feeds
 :PROPERTIES:
@@ -60,6 +62,14 @@ class Site(EMOM('site'), models.Model):
             url = url + f"#{heading.node_id}"
         return url
 
+
+    def urlize_feed(self, feed: Feed):
+        domain = self.sitedomain_set.first().domain
+        key_rest = feed.route_key.split("/", 1)[1]
+        url = f"https://{domain}/{key_rest}"
+        return url
+
+
     @classmethod
     def from_route(cls: Site, route_key: str) -> Site:
         site_key = route_key.split("/")[0]
@@ -290,6 +300,9 @@ class Feed(EMOM('feed'), models.Model):
     title = models.CharField(max_length=512)
     visibility = models.CharField(max_length=512, choices=POST_VISIBILITY)
 
+    def url(self):
+        return self.site.urlize_feed(self)
+
     @classmethod
     def create_from_arroyo(cls, doc: native.Document) -> Feed | None:
         route_key = next(iter(doc.collect_keywords("ARCOLOGY_FEED")), None)
@@ -531,6 +544,7 @@ urlpatterns = [
     path("robots.txt", views.robots),
     path("sitemap", views.sitemap, name="sitemap"),
     path("sites.css", views.site_css, name="site-css"),
+    path("feeds.json", views.feed_list, name="feed-list"),
     path("", include("django_prometheus.urls")),
     # ensure these ones are last because they're greedy!
     re_path("(?P<key>[0-9a-zA-Z/_\-]+\.xml)", views.feed, name="feed"),
@@ -609,11 +623,14 @@ def render_page(request, site, full_key):
         links = the_page.collect_links()
         page_html = the_page.to_html(links)
 
+        feeds = site.feed_set.all()
+
         page_counter.labels(page=full_key, status=200, site=site.key, agent_type=agent).inc()
 
         return render(request, "arcology/page.html", dict(
             site=site,
             page=the_page,
+            feeds=feeds,
 
             head_title=f"{the_page.title} - {site.title}",
             html_content=page_html,
@@ -754,8 +771,17 @@ Here are some [[https://medium.com/@massimo.cassandro/flexbox-separators-b284d6d
 .content > *:first-child {
     order: -1;
 }
+
+.content img {
+  display: block;
+  width: 80%;
+  margin: 0 auto;
+}
 #+end_src
 
+And some simple image wrangling:
+
+
 ** INPROGRESS Atom Feed Handler
 :PROPERTIES:
 :ID:       20240204T234814.612917
@@ -900,6 +926,30 @@ Disallow: /
 {% endfor %}
 #+end_src
 
+** Feed discovery endpoint
+:LOGBOOK:
+CLOCK: [2024-02-15 Thu 14:17]--[2024-02-15 Thu 14:41] =>  0:24
+:END:
+
+#+begin_src python :tangle arcology/views.py
+import json
+def feed_list(request):
+    site = Site.from_request(request)
+    feeds = Feed.objects.all()
+    ret = [
+        dict(
+            key=feed.route_key,
+            url=feed.site.urlize_feed(feed),
+            title=feed.title,
+            site=feed.site.key,
+            visibility=feed.visibility,
+        )
+        for feed in feeds
+    ]
+
+    return HttpResponse(json.dumps(ret), content_type="application/json")
+#+end_src
+
 ** Arcology Site Templates
 
 In short, there are four blocks that the page template and other templates will use to embed content in the rendered web page:

arcology/models.py

@@ -42,6 +42,14 @@ class Site(EMOM('site'), models.Model):
             url = url + f"#{heading.node_id}"
         return url
 
+
+    def urlize_feed(self, feed: Feed):
+        domain = self.sitedomain_set.first().domain
+        key_rest = feed.route_key.split("/", 1)[1]
+        url = f"https://{domain}/{key_rest}"
+        return url
+
+
     @classmethod
     def from_route(cls: Site, route_key: str) -> Site:
         site_key = route_key.split("/")[0]
@@ -180,6 +188,9 @@ class Feed(EMOM('feed'), models.Model):
     title = models.CharField(max_length=512)
     visibility = models.CharField(max_length=512, choices=POST_VISIBILITY)
 
+    def url(self):
+        return self.site.urlize_feed(self)
+
     @classmethod
     def create_from_arroyo(cls, doc: native.Document) -> Feed | None:
         route_key = next(iter(doc.collect_keywords("ARCOLOGY_FEED")), None)

arcology/settings/__init__.py

@@ -69,11 +69,6 @@ LOGGING = {
             "handlers": ["console"],
             "level": os.getenv("DJANGO_LOG_LEVEL", "INFO"),
             "propagate": False,
-        },
-        "generators": {
-            "handlers": ["console"],
-            "level": "DEBUG",
-            "propagate": False,
         }
     },
     "root": {

arcology/static/arcology/css/app.css

@@ -47,6 +47,12 @@ section.sidebar > div.backlinks {
 .content > *:first-child {
     order: -1;
 }
+
+.content img {
+  display: block;
+  width: 80%;
+  margin: 0 auto;
+}
 /* Org Page-specific CSS Stylings:3 ends here */
 
 /* [[file:../../../../arcology.org::*CSS][CSS:1]] */

arcology/urls.py

@@ -10,6 +10,7 @@ urlpatterns = [
     path("robots.txt", views.robots),
     path("sitemap", views.sitemap, name="sitemap"),
     path("sites.css", views.site_css, name="site-css"),
+    path("feeds.json", views.feed_list, name="feed-list"),
     path("", include("django_prometheus.urls")),
     # ensure these ones are last because they're greedy!
     re_path("(?P<key>[0-9a-zA-Z/_\-]+\.xml)", views.feed, name="feed"),

arcology/views.py

@@ -51,11 +51,14 @@ def render_page(request, site, full_key):
         links = the_page.collect_links()
         page_html = the_page.to_html(links)
 
+        feeds = site.feed_set.all()
+
         page_counter.labels(page=full_key, status=200, site=site.key, agent_type=agent).inc()
 
         return render(request, "arcology/page.html", dict(
             site=site,
             page=the_page,
+            feeds=feeds,
 
             head_title=f"{the_page.title} - {site.title}",
             html_content=page_html,
@@ -142,6 +145,25 @@ def robots(request):
     ), content_type="text/plain")
 # =robots.txt= Endpoint:1 ends here
 
+# [[file:../arcology.org::*Feed discovery endpoint][Feed discovery endpoint:1]]
+import json
+def feed_list(request):
+    site = Site.from_request(request)
+    feeds = Feed.objects.all()
+    ret = [
+        dict(
+            key=feed.route_key,
+            url=feed.site.urlize_feed(feed),
+            title=feed.title,
+            site=feed.site.key,
+            visibility=feed.visibility,
+        )
+        for feed in feeds
+    ]
+
+    return HttpResponse(json.dumps(ret), content_type="application/json")
+# Feed discovery endpoint:1 ends here
+
 # [[file:../arcology.org::*Per-Site link color dynamic CSS endpoint][Per-Site link color dynamic CSS endpoint:1]]
 def site_css(request):
     sites = Site.objects.all()

configuration.org

@@ -160,6 +160,7 @@ in {
     orgDir = "/media/org";
     folderId = "p1kld-oxnwd";
     dataDir = "/srv/arcology";
+    logLevel = "INFO";
   };
 }
 #+end_src
@@ -285,11 +286,6 @@ LOGGING = {
             "handlers": ["console"],
             "level": os.getenv("DJANGO_LOG_LEVEL", "INFO"),
             "propagate": False,
-        },
-        "generators": {
-            "handlers": ["console"],
-            "level": "DEBUG",
-            "propagate": False,
         }
     },
     "root": {

default.nix

@@ -21,8 +21,10 @@ python3.pkgs.buildPythonPackage rec {
   ]) ++ (with python3.pkgs; [
     arrow
     click
-    django
+    django_4
     django-prometheus
+    (django-stubs-ext.override { django = django_4; })
+    (django-stubs.override { django = django_4; })
     gunicorn
     polling
     setuptools

deployment.org

@@ -5,6 +5,7 @@
 #+filetags: :Project:
 
 #+ARCOLOGY_KEY: arcology/django/deploy
+#+ARCOLOGY_ALLOW_CRAWL: t
 
 * NEXT Bootstrapping on non-NixOS
 
@@ -39,8 +40,8 @@ let
 
     ARCOLOGY_DB_PATH = "${cfg.dataDir}/databases/arcology2.db";
     ARCOLOGY_ALLOWED_HOSTS = concatStringsSep "," cfg.domains;
-    # ARCOLOGY_SYNCTHING_KEY=
-    # ALLOWED_HOSTS in to settings based on cfg.domains...
+
+    ARCOLOGY_LOG_LEVEL = cfg.logLevel;
 
     GUNICORN_CMD_ARGS = "--bind=${cfg.address}:${toString cfg.port} -w ${toString cfg.workerCount}";
   };
@@ -176,13 +177,21 @@ in {
         '';
       };
 
+      logLevel = mkOption {
+        type = types.enum ["ERROR" "WARN" "INFO" "DEBUG"];
+        default = "INFO";
+        description = mdDoc ''
+          Set the Django root logging level
+        '';
+      };
+
       environmentFile = mkOption {
         type = types.path;
         default = "${cfg.dataDir}/env";
         description = mdDoc ''
           A file containing environment variables you may not want to put in the nix store.
 
-          For example, put a syncthing key in there:
+          For example, you could put a syncthing key in there:
 
           ARCOLOGY_SYNCTHING_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA;
         '';
@@ -241,12 +250,16 @@ in {
 probably just =getFlake= but augh.
 
 *** NEXT service hardening
-*** INPROGRESS static files under gunicorn/nginx
+*** DONE static files under gunicorn/nginx
 :LOGBOOK:
+- State "DONE"       from "INPROGRESS" [2024-02-15 Thu 19:44]
 - State "INPROGRESS" from "NEXT"       [2024-02-15 Thu 12:08]
 CLOCK: [2024-02-15 Thu 12:08]--[2024-02-15 Thu 12:58] =>  0:50
 :END:
-*** NEXT secret infrastructure for the syncthing key
+*** DONE secret infrastructure for the syncthing key
+:LOGBOOK:
+- State "DONE"       from "NEXT"       [2024-02-15 Thu 19:44]
+:END:
 
 or a way to load that in to the DB 🤔
 

flake.lock

@@ -6,11 +6,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1707114472,
-        "narHash": "sha256-yYDz01/0m1ObUU6ULCg1itjKnJWMu02GhG0E4etMA+0=",
+        "lastModified": 1708042667,
+        "narHash": "sha256-4QcfGV2A3VsR8d3qGvXQSrb1iKssS7tMTV2FCEA0eXc=",
         "ref": "refs/heads/main",
-        "rev": "f890e941bbcb4e3402a8c23464373badf1180065",
-        "revCount": 148,
+        "rev": "fa6bbaf8d3044f62beb3435d4efd6b2ee6bdae46",
+        "revCount": 155,
         "type": "git",
         "url": "https://code.rix.si/rrix/arroyo"
       },

interfaces.org

@@ -69,11 +69,13 @@ It finds any =.org= file which are regular files, not symlinks.
 #+begin_src python :tangle roam/management/commands/ingestfiles.py
         files_glob = "**/*.org"
         files = glob.glob(files_glob, root_dir=src, recursive=True)
-        files = map(lambda it: str(pathlib.Path(src).joinpath(it).expanduser()), files)
-        files = filter(os.path.isfile, files)
-        files = list(filterfalse(os.path.islink, files))
+        files = map(lambda it: pathlib.Path(src).joinpath(it).expanduser(), files)
+        files = filter(lambda it: it.is_file(), files)
+        files = filter(lambda it: not it.is_symlink(), files)
+        files = map(lambda it: str(it), files)
+        files = list(files)
 
-        logger.info(f"Collected {len(files)} files for parsing.")
+        logger.info(f"Collected {len(list(files))} files for parsing.")
 #+end_src
 
 It goes over each of these, checking whether the document has been ingested before, or if it needs to be updated, or if it is an invalid document that must be skipped. =roam.core.should_file_persist= returns a tuple with the state needed to decide what to do with a document, and then that action is taken here.
@@ -111,6 +113,7 @@ Finally, as mentioned, the ordered =ARROYO_EXTRACTORS= are run.
 
 #+begin_src python :tangle roam/management/commands/ingestfiles.py
         for path in sorter.static_order():
+            logger.debug(f"Attempting to parse {path}.")
             doc = parse_doc(path)
             arroyo_persist_one_file(doc)
         logger.info(f"Scanned {len(files)} and parsed/persisted {len(docs)} docs.")
@@ -375,22 +378,23 @@ The functionality to query the Syncthing [[https://docs.syncthing.net/rest/event
 For each file path, it checks that the file name is not a temporary file name, that it ends with =.org= and the change is made in the folder we are monitoring. Anything else is elided.
 
 #+begin_src python :tangle syncthonk/management/commands/watchsync.py
-            ingest = True
+            ingest_this = True
             if file_path.name.startswith(".#"):
                 logger.debug("skip temp file")
-                ingest = False
+                ingest_this = False
             elif file_path.suffix != ".org":
-                logger.debug("skip non-org file")
-                ingest = False
+                logger.debug(f"skip non-org file {file_path.suffix}")
+                ingest_this = False
             elif event_folder_id != self.folder_id:
-                logger.debug("skip unmonitored folder")
-                ingest = False
+                logger.debug(f"skip unmonitored folder {event_folder_id}")
+                ingest_this = False
             # add new failure cases here.
-            logger.debug(f"proc {last_since} {event_folder_id}, ingest? {ingest}")
-            if ingest == True:
+            logger.debug(f"proc {last_since} {event_folder_id}, ingest? {ingest}, ingest_this? {ingest_this}")
+            if ingest_this == True:
+                ingest = ingest_this
                 logger.debug(f"{event}")
         if ingest:
-            call_command("ingestfiles", self.expanded_path)
+          call_command('ingestfiles', self.expanded_path)
         return last_since
 #+end_src
 

roam/management/commands/ingestfiles.py

@@ -43,11 +43,13 @@ class Command(BaseCommand):
 # [[file:../../../interfaces.org::*Ingest files in bulk using the command line][Ingest files in bulk using the command line:5]]
         files_glob = "**/*.org"
         files = glob.glob(files_glob, root_dir=src, recursive=True)
-        files = map(lambda it: str(pathlib.Path(src).joinpath(it).expanduser()), files)
-        files = filter(os.path.isfile, files)
-        files = list(filterfalse(os.path.islink, files))
+        files = map(lambda it: pathlib.Path(src).joinpath(it).expanduser(), files)
+        files = filter(lambda it: it.is_file(), files)
+        files = filter(lambda it: not it.is_symlink(), files)
+        files = map(lambda it: str(it), files)
+        files = list(files)
 
-        logger.info(f"Collected {len(files)} files for parsing.")
+        logger.info(f"Collected {len(list(files))} files for parsing.")
 # Ingest files in bulk using the command line:5 ends here
 
 # [[file:../../../interfaces.org::*Ingest files in bulk using the command line][Ingest files in bulk using the command line:6]]
@@ -79,6 +81,7 @@ class Command(BaseCommand):
 
 # [[file:../../../interfaces.org::*Ingest files in bulk using the command line][Ingest files in bulk using the command line:8]]
         for path in sorter.static_order():
+            logger.debug(f"Attempting to parse {path}.")
             doc = parse_doc(path)
             arroyo_persist_one_file(doc)
         logger.info(f"Scanned {len(files)} and parsed/persisted {len(docs)} docs.")

scaffolding.org

@@ -72,8 +72,10 @@ python3.pkgs.buildPythonPackage rec {
   ]) ++ (with python3.pkgs; [
     arrow
     click
-    django
+    django_4
     django-prometheus
+    (django-stubs-ext.override { django = django_4; })
+    (django-stubs.override { django = django_4; })
     gunicorn
     polling
     setuptools
@@ -90,6 +92,46 @@ python3.pkgs.buildPythonPackage rec {
 }
 #+end_src
 
+** Dev Environment
+
+=nix develop= or =nix-shell= will set you up with an environment that has Python programming dependencies available.
+
+#+begin_src nix :tangle shell.nix
+{ pkgs ? import <nixpkgs> {},
+  python3 ? pkgs.python3,
+
+  arroyo_rs ? pkgs.callPackage /home/rrix/org/arroyo/default.nix {},
+}:
+let
+  myPython = python3.withPackages( pp: with pp; [
+    pip
+    pytest
+    mypy
+
+    arrow
+    arroyo_rs
+    django_4
+    django-prometheus
+    (django-stubs-ext.override { django = django_4; })
+    (django-stubs.override { django = django_4; })
+    gunicorn
+    polling
+  ]);
+in pkgs.mkShell {
+  packages = (with pkgs; [
+    maturin
+    myPython
+
+    pyright
+    black]);
+  RUST_SRC_PATH = "${pkgs.rust.packages.stable.rustPlatform.rustLibSrc}";
+  NIX_CONFIG = "builders =";
+  shellHook = ''
+    PYTHONPATH=${myPython}/${myPython.sitePackages}
+  '';
+}
+#+end_src
+
 ** A Flake to tie everything together and make it possible to run remotely
 
 Nix is really going this direction, I'm not sure it's worthwhile but I'm going to see how to adapt to this world. It should be possible to =nix run= a few apps to be able to operate the arcology.
@@ -139,46 +181,6 @@ Nix is really going this direction, I'm not sure it's worthwhile but I'm going t
 }
 #+end_src
 
-** Dev Environment
-
-=nix develop= or =nix-shell= will set you up with an environment that has Python programming dependencies available.
-
-#+begin_src nix :tangle shell.nix
-{ pkgs ? import <nixpkgs> {},
-  python3 ? pkgs.python3,
-
-  arroyo_rs ? pkgs.callPackage /home/rrix/org/arroyo/default.nix {},
-}:
-let
-  myPython = python3.withPackages( pp: with pp; [
-    pip
-    pytest
-    mypy
-
-    arrow
-    arroyo_rs
-    django_4
-    django-prometheus
-    django-stubs-ext.override { django = django_4; }
-    django-stubs.override { django = django_4; }
-    gunicorn
-    polling
-  ]);
-in pkgs.mkShell {
-  packages = (with pkgs; [
-    maturin
-    myPython
-
-    pyright
-    black]);
-  RUST_SRC_PATH = "${pkgs.rust.packages.stable.rustPlatform.rustLibSrc}";
-  NIX_CONFIG = "builders =";
-  shellHook = ''
-    PYTHONPATH=${myPython}/${myPython.sitePackages}
-  '';
-}
-#+end_src
-
 ** Direnv
 
 [[id:45fc2a02-fcd0-40c6-a29e-897c0ee7b1c7][direnv]] fucking rules.

shell.nix

@@ -14,8 +14,8 @@ let
     arroyo_rs
     django_4
     django-prometheus
-    django-stubs-ext.override { django = django_4; }
-    django-stubs.override { django = django_4; }
+    (django-stubs-ext.override { django = django_4; })
+    (django-stubs.override { django = django_4; })
     gunicorn
     polling
   ]);

syncthonk/management/commands/watchsync.py

@@ -93,21 +93,22 @@ class Command(BaseCommand):
 # Ingest files on-demand using Syncthing:6 ends here
 
 # [[file:../../../interfaces.org::*Ingest files on-demand using Syncthing][Ingest files on-demand using Syncthing:7]]
-            ingest = True
+            ingest_this = True
             if file_path.name.startswith(".#"):
                 logger.debug("skip temp file")
-                ingest = False
+                ingest_this = False
             elif file_path.suffix != ".org":
-                logger.debug("skip non-org file")
-                ingest = False
+                logger.debug(f"skip non-org file {file_path.suffix}")
+                ingest_this = False
             elif event_folder_id != self.folder_id:
-                logger.debug("skip unmonitored folder")
-                ingest = False
+                logger.debug(f"skip unmonitored folder {event_folder_id}")
+                ingest_this = False
             # add new failure cases here.
-            logger.debug(f"proc {last_since} {event_folder_id}, ingest? {ingest}")
-            if ingest == True:
+            logger.debug(f"proc {last_since} {event_folder_id}, ingest? {ingest}, ingest_this? {ingest_this}")
+            if ingest_this == True:
+                ingest = ingest_this
                 logger.debug(f"{event}")
         if ingest:
-            call_command("ingestfiles", self.expanded_path)
+          call_command('ingestfiles', self.expanded_path)
         return last_since
 # Ingest files on-demand using Syncthing:7 ends here