deploy local API instance of the arcology as user-unit

README.org

@@ -53,6 +53,7 @@ This project is a living document. It's [[id:cce/literate_programming][Literate
 - [[id:arcology/django/config][Arcology Project configuration]] describes and implements the knobs the user can twiddle to configure the Arcology's operation
 - [[id:arcology/django/interfaces][Interfacing with the Arcology]] provides a set of management commands and a Syncthing client which will automatically ingest new files in to the database. This document describes all the tools a prospective user of the CCE will need to use for surface-level features.
 - [[id:20240213T124300.774781][Deploying the Arcology]] lays out the basics of operating a Wobserver, but also provides a methodology for bootstrapping the Arcology on systems not already running NixOS.
+- [[id:20240313T153901.656967][A Localhost API for the Arcology]] lets one make quick queries of the Arcology's metadata to [[id:128ab0e8-a1c7-48bf-9efe-0c23ce906a48][turn Org Mode in to Hypermedia]].
 - [[id:20240226T132507.817450][The Arcology's Site Maps and Discovery Mechanisms]] show you how you can discover content published in an Arcology.
 - The [[id:arcology/django/scaffolding][Arcology Project Scaffolding]] contains the files necessary to run the project, defines the nix environment, and the python environment, and the base Django apparatus.
 - The [[id:arcology/django/roam][Arcology Roam Models]] are the base metadata of the [[id:cce/org-roam][org-roam]] documents

arcology.org

@@ -540,6 +540,7 @@ urlpatterns = [
     path("feeds.json", views.feed_list, name="feed-list"),
     path("", include("django_prometheus.urls")),
     path("", include("sitemap.urls")),
+    path("api/v1/", include("localapi.urls")),
     # ensure these ones are last because they're greedy!
     re_path("(?P<key>[0-9a-zA-Z/_\-]+\.xml)", views.feed, name="feed"),
     re_path("(?P<key>[0-9a-zA-Z/_\-]+)", views.org_page, name="org-page"),
@@ -855,6 +856,30 @@ These rules annotate task headings by inserting an icon before them.
 }
 #+end_src
 
+This will display the header arguments to =org-babel= source blocks: You're staring right at one!
+
+#+begin_src css :tangle arcology/static/arcology/css/app.css :mkdirp yes
+span.babel-args {
+    text-align: right;
+    display: block;
+    background: var(--light-gray);
+    margin-bottom: 0;
+}
+
+pre.src {
+    border-top: 1px solid var(--black);
+    background-color: var(--light-gray);
+    font-style: normal;
+    overflow: scroll;
+    margin-top: 0;
+
+    padding-top: 1em;
+    padding-left: 0.5em;
+    padding-bottom: 1em;
+    padding-right: 0.5em;
+}
+#+end_src
+
 ** Atom Feed Handler
 :PROPERTIES:
 :ID:       20240204T234814.612917

arcology/management/commands/seed.py

@@ -16,6 +16,8 @@ class Command(BaseCommand):
     requires_migrations_checks = True
 
     def handle(self, *args, **kwargs):
+        generator_roles = generators.models.GeneratorRole.get_or_create_many(settings.ARROYO_ROLES)
+
         sites_path = settings.BASE_DIR / "arcology/settings/sites.json"
         with open(sites_path, "r") as f:
           sites = json.load(f)

arcology/settings/__init__.py

@@ -6,7 +6,7 @@ from .generators import *
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent.parent
-DATABASE_PATH = os.getenv("ARCOLOGY_DB_PATH", pathlib.Path(os.getcwd()) / "db.sqlite3")
+DATABASE_PATH = pathlib.Path(os.getenv("ARCOLOGY_DB_PATH", pathlib.Path(os.getcwd()) / "db.sqlite3")).expanduser()
 # Service Configuration:1 ends here
 
 # [[file:../../configuration.org::*Environment Variables][Environment Variables:1]]
@@ -23,7 +23,7 @@ SYNCTHING_KEY = os.getenv("ARCOLOGY_SYNCTHING_KEY")
 # Environment Variables:3 ends here
 
 # [[file:../../configuration.org::*Environment Variables][Environment Variables:4]]
-BASE_CACHE_PATH = os.environ.get("ARCOLOGY_CACHE_PATH", '/var/tmp/django_cache')
+BASE_CACHE_PATH = pathlib.Path(os.environ.get("ARCOLOGY_CACHE_PATH", '/var/tmp/django_cache')).expanduser()
 CACHES = {
     'default': {
         'BACKEND': 'django_prometheus.cache.backends.filebased.FileBasedCache',
@@ -32,6 +32,10 @@ CACHES = {
 }
 # Environment Variables:4 ends here
 
+# [[file:../../configuration.org::*Environment Variables][Environment Variables:5]]
+LOCALAPI_BEARER_TOKEN = os.environ.get("ARCOLOGY_LOCALAPI_BEARER_TOKEN", "changeme!")
+# Environment Variables:5 ends here
+
 # [[file:../../configuration.org::*Hostname configuration from =arcology.model.Site=, eventually][Hostname configuration from =arcology.model.Site=, eventually:1]]
 ALLOWED_HOSTS = "thelionsrear.com,rix.si,arcology.garden,whatthefuck.computer,cce.whatthefuck.computer,cce.rix.si,engine.arcology.garden,127.0.0.1,localhost,v2.thelionsrear.com,v2.arcology.garden,cce2.whatthefuck.computer,engine2.arcology.garden".split(',')
 # Hostname configuration from =arcology.model.Site=, eventually:1 ends here
@@ -44,6 +48,7 @@ INSTALLED_APPS = [
     "generators", # [[id:arroyo/django/generators][The Arroyo Generators]]
     "syncthonk", # [[id:20231218T183551.765340][Arcology watchsync Command]]
     "sitemap", # [[id:20240226T132507.817450][The Arcology's Site Maps and Discovery Mechanisms]]
+    "localapi", # [[id:20240313T153901.656967][A Localhost API for the Arcology]]
     
     "django_htmx",
     "django_prometheus",

arcology/settings/generators.py

@@ -1,7 +1,8 @@
 # [[file:../../configuration.org::*directory configuration][directory configuration:1]]
 import pathlib
+import os
 
-ARCOLOGY_BASE_DIR = str(pathlib.Path("~/org").expanduser())
+ARCOLOGY_BASE_DIR = pathlib.Path(os.environ.get("ARCOLOGY_BASE_DIR", "~/org")).expanduser()
 ARCOLOGY_EMACS_SNIPPETS_DIR = str(pathlib.Path("~/org/cce/").expanduser())
 ARROYO_BASE_DIR = str(pathlib.Path("~/arroyo-nix").expanduser())
 # directory configuration:1 ends here

arcology/static/arcology/css/app.css

@@ -71,6 +71,28 @@ section.sidebar > div.backlinks {
 }
 /* Org Page-specific CSS Stylings:5 ends here */
 
+/* [[file:../../../../arcology.org::*Org Page-specific CSS Stylings][Org Page-specific CSS Stylings:6]] */
+span.babel-args {
+    text-align: right;
+    display: block;
+    background: var(--light-gray);
+    margin-bottom: 0;
+}
+
+pre.src {
+    border-top: 1px solid var(--black);
+    background-color: var(--light-gray);
+    font-style: normal;
+    overflow: scroll;
+    margin-top: 0;
+
+    padding-top: 1em;
+    padding-left: 0.5em;
+    padding-bottom: 1em;
+    padding-right: 0.5em;
+}
+/* Org Page-specific CSS Stylings:6 ends here */
+
 /* [[file:../../../../arcology.org::*CSS][CSS:1]] */
 body {
     font-family: "Vulf Mono", monospace;

arcology/urls.py

@@ -13,6 +13,7 @@ urlpatterns = [
     path("feeds.json", views.feed_list, name="feed-list"),
     path("", include("django_prometheus.urls")),
     path("", include("sitemap.urls")),
+    path("api/v1/", include("localapi.urls")),
     # ensure these ones are last because they're greedy!
     re_path("(?P<key>[0-9a-zA-Z/_\-]+\.xml)", views.feed, name="feed"),
     re_path("(?P<key>[0-9a-zA-Z/_\-]+)", views.org_page, name="org-page"),

configuration.org

@@ -174,6 +174,30 @@ The NixOS module which defines =services.arcology-ng= is in [[id:20240213T124300
 
 ** NEXT the package import needs to be much better than this.
 
+* Deploy manifests for [[id:cce/home-manager][a Dynamic Home Manager Configuration]]
+
+This deploys [[id:20240313T153901.656967][A Localhost API for the Arcology]] to any Home Manager user using =systemd= User Units:
+
+#+begin_src nix :tangle ~/arroyo-nix/hm/arcology-localapi.nix
+{ pkgs, ... }:
+
+let
+  arroyo_rs = pkgs.callPackage /home/rrix/org/arroyo/default.nix {};
+  arcology = pkgs.callPackage /home/rrix/org/arcology-django/default.nix { inherit arroyo_rs; };
+in {
+  imports = [ ./arcology-localapi-mod.nix ];
+  home.packages = [ arcology ];
+
+  services.arcology2 = {
+    enable = true;
+    packages.arcology = arcology;
+    folderId = "p1kld-oxnwd";
+    environmentFile = "/home/rrix/sync/private-files/.arcology-env";
+  };
+}
+#+end_src
+
+
 * Service Configuration
 :PROPERTIES:
 :ID:       20231217T155611.177995
@@ -204,7 +228,7 @@ from .generators import *
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent.parent
-DATABASE_PATH = os.getenv("ARCOLOGY_DB_PATH", pathlib.Path(os.getcwd()) / "db.sqlite3")
+DATABASE_PATH = pathlib.Path(os.getenv("ARCOLOGY_DB_PATH", pathlib.Path(os.getcwd()) / "db.sqlite3")).expanduser()
 #+END_SRC
 
 ** Environment Variables
@@ -231,7 +255,7 @@ SYNCTHING_KEY = os.getenv("ARCOLOGY_SYNCTHING_KEY")
 The =ARCOLOGY_CACHE_PATH= is set to a path that multi-process django can use to cache processed HTML and Atom between processes.
 
 #+BEGIN_SRC python :tangle arcology/settings/__init__.py
-BASE_CACHE_PATH = os.environ.get("ARCOLOGY_CACHE_PATH", '/var/tmp/django_cache')
+BASE_CACHE_PATH = pathlib.Path(os.environ.get("ARCOLOGY_CACHE_PATH", '/var/tmp/django_cache')).expanduser()
 CACHES = {
     'default': {
         'BACKEND': 'django_prometheus.cache.backends.filebased.FileBasedCache',
@@ -240,6 +264,10 @@ CACHES = {
 }
 #+END_SRC
 
+#+begin_src python :tangle arcology/settings/__init__.py
+LOCALAPI_BEARER_TOKEN = os.environ.get("ARCOLOGY_LOCALAPI_BEARER_TOKEN", "changeme!")
+#+end_src
+
 ** NEXT Hostname configuration from =arcology.model.Site=, eventually
 
 When I have the sites organized in an org-mode table, i'll reapproach the hostname list, and probably before then when i want to test domain-based routing.
@@ -258,6 +286,7 @@ basically, each org file in this repository, and maybe one or two of your own, a
 | "generators" | [[id:arroyo/django/generators][The Arroyo Generators]]                             |
 | "syncthonk"  | [[id:20231218T183551.765340][Arcology watchsync Command]]                        |
 | "sitemap"    | [[id:20240226T132507.817450][The Arcology's Site Maps and Discovery Mechanisms]] |
+| "localapi"   | [[id:20240313T153901.656967][A Localhost API for the Arcology]]                  |
 
 #+BEGIN_SRC python :tangle arcology/settings/__init__.py :noweb yes
 # Application definition
@@ -327,8 +356,9 @@ LOGGING = {
 
 #+BEGIN_SRC python :tangle arcology/settings/generators.py :noweb yes
 import pathlib
+import os
 
-ARCOLOGY_BASE_DIR = str(pathlib.Path("~/org").expanduser())
+ARCOLOGY_BASE_DIR = pathlib.Path(os.environ.get("ARCOLOGY_BASE_DIR", "~/org")).expanduser()
 ARCOLOGY_EMACS_SNIPPETS_DIR = str(pathlib.Path("~/org/cce/").expanduser())
 ARROYO_BASE_DIR = str(pathlib.Path("~/arroyo-nix").expanduser())
 #+END_SRC

deployment.org

@@ -35,7 +35,7 @@ let
 
   env = {
     ARCOLOGY_ENVIRONMENT = cfg.environment;
-    ARCOLOGY_DIRECTORY = cfg.orgDir;
+    ARCOLOGY_BASE_DIR = cfg.orgDir;
     ARCOLOGY_STATIC_ROOT = cfg.staticRoot;
 
     ARCOLOGY_DB_PATH = "${cfg.dataDir}/databases/arcology2.db";
@@ -80,7 +80,7 @@ let
         Restart="on-failure";
         RestartSec=5;
         RestartSteps=10;
-        RestartMaxDelay="1min";
+        RestartMaxDelaySec="1min";
         # hardening...
       };
     };
@@ -89,6 +89,9 @@ let
       after = ["network.target"];
       wantedBy = ["multi-user.target"];
       environment = env;
+      preStart = ''
+        find ${cfg.multiProcDir} -type f -delete
+      '';
       script = ''
         ${pyenv}/bin/python -m gunicorn arcology.wsgi
       '';
@@ -102,7 +105,7 @@ let
         Restart="on-failure";
         RestartSec=5;
         RestartSteps=10;
-        RestartMaxDelay="1min";
+        RestartMaxDelaySec="1min";
         # hardening...
       };
     };
@@ -206,9 +209,11 @@ in {
         description = mdDoc ''
           A file containing environment variables you may not want to put in the nix store.
 
-          For example, you could put a syncthing key in there:
+          For example, you could put a syncthing key 
+          and a bearer token for the Local API in there:
 
           ARCOLOGY_SYNCTHING_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA;
+          ARCOLOGY_LOCALAPI_BEARER_TOKEN=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA;
         '';
       };
 

flake.lock

@@ -6,11 +6,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1710219899,
-        "narHash": "sha256-IGWuukeJ7egGscXKDAksdPCdftqnQfsHvzQMSYq/Q84=",
+        "lastModified": 1710377466,
+        "narHash": "sha256-7p5/TWrVgK5/DeQRKT7TgbASAzTT4ctxyvK4rkIvrf8=",
         "ref": "refs/heads/main",
-        "rev": "1b7e71d9e6b459fbdddcd51b04bda20085e90959",
-        "revCount": 161,
+        "rev": "0571f3369c7ee5b36ea3acd2eba1d8dd6de6115f",
+        "revCount": 162,
         "type": "git",
         "url": "https://code.rix.si/rrix/arroyo"
       },

interfaces.org

@@ -1,4 +1,5 @@
 # -*- org-src-preserve-indentation: t; -*-
+#+filetags: :Project:
 :PROPERTIES:
 :ID:       arcology/django/interfaces
 :ROAM_ALIASES: "The Arcology Management Commands"
@@ -212,38 +213,23 @@ roam/management/commands/__init__.py (this isn't needed)
 These will be the "public" interfaces for the generators, a set of Emacs functions that can be loaded in to the environment to call out to the management commands:
 
 #+begin_src emacs-lisp :tangle ~/org/cce/arcology-django-helpers.el :results none
-(defun arroyo-collect-files (module &optional role)
-  (setenv "ARCOLOGY_DB_PATH" "/home/rrix/org/arcology-django/db.sqlite3")
-  (let* ((flake-path "path:/home/rrix/org/arcology-django")
-         (cmd (s-join " " (list (format "nix run %s#arcology" flake-path)
-                                "--"
-                                "generate -m"
-                                module
-                                (when role
-                                  (format "-r %s" role))
-                                "--dry-run"
-                                "2>/dev/null"
-                                " | sort")))
-         (output (shell-command-to-string cmd)))
-    output))
+;; (defun arroyo-collect-files (module &optional role)
+;;   (setenv "ARCOLOGY_DB_PATH" "/home/rrix/org/arcology-django/db.sqlite3")
+;;   (let* ((flake-path "path:/home/rrix/org/arcology-django")
+;;          (cmd (s-join " " (list (format "nix run %s#arcology" flake-path)
+;;                                 "--"
+;;                                 "generate -m"
+;;                                 module
+;;                                 (when role
+;;                                   (format "-r %s" role))
+;;                                 "--dry-run"
+;;                                 "2>/dev/null"
+;;                                 " | sort")))
+;;          (output (shell-command-to-string cmd)))
+;;     output))
 
 (defun arroyo-generate-imports (module &optional role destination  do-sort)
-  (setenv "ARCOLOGY_DB_PATH" "/home/rrix/org/arcology-django/db.sqlite3")
-  (let* ((flake-path "path:/home/rrix/org/arcology-django")
-         (do-sort (and do-sort t))
-         (cmd (s-join " " (list (format "nix run %s#arcology" flake-path)
-                                "--"
-                                "generate -m"
-                                module
-                                (when role
-                                  (format "-r %s" role))
-                                (when destination
-                                  (format "-d %s" destination))
-                                "2>/dev/null"
-                                (when do-sort
-                                  " | sort"))))
-         (output (shell-command-to-string cmd)))
-    output))
+  (arcology-api-generator module role destination do-sort))
 
 (defun arroyo-generate-imports-formatted (format-fn output)
   (thread-last (arroyo-generate-imports "home-manager" "server")
@@ -279,14 +265,16 @@ from django.conf import settings
 
 import json
 import pathlib
-import urllib.parse 
-import urllib.request 
+import urllib.parse
+import urllib.request
 
 import polling
 
 import logging
 logger = logging.getLogger(__name__)
 # logger.setLevel(logging.DEBUG)
+
+import roam.models
 #+end_src
 
 The command goes in to a simple loop with [[https://github.com/justiniso/polling][=polling=]] used to ensure that transient errors are retried and the loop does not die without good cause. The [[id:cce/syncthing][Syncthing]] API key is loaded from the [[id:arcology/django/config][Project Configuration]] and the path to monitor is passed in as an argument. Note that this requires the path to match a Syncthing directory! If you are monitoring a folder which is subdirectory of a shared folder, make sure to pass the root of the shared folder in instead or break the path in to its own Syncthing share.
@@ -315,7 +303,7 @@ class Command(BaseCommand):
             self.folder_id = self.get_folder_id_for_path(self.expanded_path)
             logger.info(f"fetched folder ID {self.folder_id} AKA {self.expanded_path} from syncthing API")
         #+end_src
-        
+
 it would be nice to get the pagination key from the DB to have persistence across invocations, but re-running the ingester on startup when there isn't necessarily changes isn't really that bad, it'll just cause some disk IO while it checks file hashes:
 
 #+begin_src python :tangle syncthonk/management/commands/watchsync.py
@@ -372,7 +360,8 @@ The functionality to query the Syncthing [[https://docs.syncthing.net/rest/event
         last_since = None
         for event in jason:
             last_since = event.get("id")
-            event_folder_id = event.get("data", {}).get("folder", "")
+            data = event.get("data", {})
+            event_folder_id = data.get("folder", "")
             file_path = pathlib.Path(event.get("data", {}).get("path"))
 #+end_src
 
@@ -389,8 +378,25 @@ For each file path, it checks that the file name is not a temporary file name, t
             elif event_folder_id != self.folder_id:
                 logger.debug(f"skip unmonitored folder {event_folder_id}")
                 ingest_this = False
+            #+end_src
+
+If a file is deleted, make sure it's removed from the database. Note that this is still liable to miss files; eventually I will want to add a management command to go over the whole file listing and true up the database, but this is good enough for now as long as it's kept to be realtime.
+
+            #+begin_src python :tangle syncthonk/management/commands/watchsync.py
+            elif data.get("action") == "deleted":
+                final_path = self.expanded_path.joinpath(file_path)
+                try:
+                    f = roam.models.File.objects.get(path=final_path)
+                    logger.debug(f"deleting {final_path}!")
+                    f.delete()
+                    ingest_this = False
+                except roam.models.File.DoesNotExist:
+                    pass
+            #+end_src
+
+            #+begin_src python :tangle syncthonk/management/commands/watchsync.py
             # add new failure cases here.
-            logger.debug(f"proc {last_since} {event_folder_id}, ingest? {ingest}, ingest_this? {ingest_this}")
+            logger.debug(f"proc {last_since} {event_folder_id}, ingest? {ingest}, ingest_this? {ingest_this}: {json.dumps(event)}")
             if ingest_this == True:
                 ingest = ingest_this
                 logger.debug(f"{event}")
@@ -401,6 +407,11 @@ For each file path, it checks that the file name is not a temporary file name, t
 
 The command returns the last event ID of the request which is used to paginate future requests made in the loop.
 
+** DONE make sure we are able to handle deletions here...
+:LOGBOOK:
+- State "DONE"       from "NEXT"       [2024-03-13 Wed 15:37]
+:END:
+
 * Create Base Sites
 :PROPERTIES:
 :ID:       20231217T154835.232283
@@ -429,6 +440,8 @@ class Command(BaseCommand):
     requires_migrations_checks = True
 
     def handle(self, *args, **kwargs):
+        generator_roles = generators.models.GeneratorRole.get_or_create_many(settings.ARROYO_ROLES)
+
         sites_path = settings.BASE_DIR / "arcology/settings/sites.json"
         with open(sites_path, "r") as f:
           sites = json.load(f)

localapi.org

@@ -0,0 +1,509 @@
+:PROPERTIES:
+:ID:       20240313T153901.656967
+:END:
+#+TITLE: A Localhost API for the Arcology
+#+FILETAGS: :Project:
+
+#+ARCOLOGY_KEY: arcology/localapi
+#+ARROYO_EMACS_MODULE: arcology-localapi-commands
+#+AUTO_TANGLE: t
+
+By moving the database out of EmacSQL and in to Django, I have hobbled some of the Arcology's [[id:knowledge_base][Knowledge Management]] [[id:e79d4cdc-082f-4b1d-ae08-d979802f09ee][Living Systems]] and meta-cognitive skills. =arroyo-db-query= no longer existing means that things like the [[id:20230526T105534.711282][Direnv arroyo-db integration]] stopped working, and the helpers in [[id:cce/my_nixos_configuration][My NixOS configuration]] and elsewhere rely on the stringly-typed [[id:20231217T154938.132553][Arcology generate Command]] and a =nix run= invocation to function.
+
+I think it's worth building a small HTTP JSON API which can serve some of the common queries, to at least see if it's worth the trouble..
+
+Here's how to use this from Emacs Lisp:
+
+- =(arcology-localapi-call method path)= is an API helper which given an API path will fetch the data and return a deserialized JSON structure
+- these interactive commands will fetch a URL, putting it on your kill ring or clipboard if you call it with =M-x= or equivalent:
+- =(arcology-key-to-url page-key &optional heading-id)= will take an =ARCOLOGY_KEY= and return a URL
+- =(arcology-file-to-url file-path &optional heading-id)= will do the same with a file path from =(buffer-file-name)= or so.
+- =(arcology-url-at-point)= gets the org-id of the heading your cursor is in if it has one, and makes a URL that links directly to that.
+- =(arcology-read-url)= pops up a list of all the org-roam headings, and returns a URL to it.
+
+* Server view and Arroyo Emacs lisp scaffolding
+
+I probably should use Django REST Framework for this, but. I'm just gonna beat two stones together until a JSON API falls out. This will probably change the first time I add a =POST= call or want to do more complex auth stuff.
+
+The API will be simple, with a bearer token provided by a local state file:
+
+#+begin_src emacs-lisp :tangle ~/org/cce/arcology-localapi-commands.el
+(use-package plz)
+
+(defun arcology-fetch-localapi-bearer-token ()
+  (interactive)
+  (customize-save-variable
+   'arcology-localapi-bearer-token
+   (or (getenv "ARCOLOGY_LOCALAPI_BEARER_TOKEN")
+       (thread-last
+         ". ~/sync/private-files/.arcology-env && echo $ARCOLOGY_LOCALAPI_BEARER_TOKEN"
+         (shell-command-to-string)
+         (s-chop-suffix "\n" )))))
+
+(defcustom arcology-api-base "http://127.0.0.1:29543/api/v1"
+  "localapi base url")
+
+(defun arcology-localapi-call (method path &rest plz-args)
+  (let ((plz-args (or plz-args '(:as json-read))))
+    (apply 'plz method (format "%s%s" arcology-api-base path)
+             :headers `(("Authorization" . ,(format "Bearer %s" arcology-localapi-bearer-token)))
+             plz-args)))
+#+end_src
+
+These are the API URLs and basic imports... nothing to concern yourself with, probably!
+
+#+begin_src python :tangle localapi/urls.py :mkdirp yes
+from django.contrib import admin
+from django.urls import path, re_path, include
+
+from localapi import views
+
+urlpatterns = [
+    path("", views.index),
+    path("generate/<slug:module>/<slug:role>", views.generate, name="keyword_by_key"),
+    path("generate/<slug:module>", views.generate, name="keyword_by_key"),
+    path("keywords/<slug:key>", views.keyword_by_key, name="keyword_by_key"),
+    re_path("keywords/(?P<key>[0-9a-zA-Z_-]+)/(?P<value>[0-9a-zA-Z/_\-]+)", views.keyword_by_key_value, name="keyword_by_key"),
+    re_path("page/(?P<route_key>[0-9a-zA-Z/_\-]+)", views.page_metadata, name="page_metadata"),
+    re_path("file/(?P<file_path>[0-9a-zA-Z/_\-\.]+)", views.file_metadata, name="file_metadata"),
+]
+#+end_src
+
+#+begin_src python :tangle localapi/views.py
+from django.http import HttpRequest, HttpResponse, HttpResponseNotFound, Http404, JsonResponse
+from django.shortcuts import render, get_object_or_404
+
+from arcology.models import Page, Feed, Site
+from roam.models import Link
+from localapi.auth import authenticated
+
+from prometheus_client import Counter, Histogram
+
+import logging
+logger = logging.getLogger(__name__)
+#+end_src
+
+The API is fairly simple and built-to-purpose, we start with an endpoint that can be queried to validate that the API Bearer token is valid:
+
+#+begin_src python :tangle localapi/views.py
+@authenticated
+def index(request):
+    return JsonResponse(dict(state="ok :)"))
+#+end_src
+
+** DONE Local API Auth Middleware
+:PROPERTIES:
+:ID:       20240313T170425.687381
+:END:
+:LOGBOOK:
+- State "DONE"       from              [2024-03-13 Wed 17:01]
+:END:
+
+This API should be secured from CSRF and local scanners, we'll just put a bearer token on it. Since the API is meant to only be running on a CCE endpoint, and not on [[id:20211120T220054.226284][The Wobserver]] the API authentication can be smoothed out a bit and hopefully made transparent to the user. The Bearer token is set in the [[id:arcology/django/config][Arcology Project Configuration]], and can be overridden in an environment variable =ARCOLOGY_LOCALAPI_BEARER_TOKEN=. The NixOS endpoint configuration module for this API will include token generation and placement so that this should be transparent to a local command user. Decorating a view with =@authenticated= will add this bearer token authentication to the view.
+
+#+begin_src python :tangle localapi/auth.py
+import re
+from django.conf import settings
+from django.http import HttpRequest, JsonResponse
+
+def authenticate_request(request: HttpRequest):
+    r = re.compile(r'Bearer (\S+)')
+    bearer = request.headers.get("Authorization", "")
+    match = r.match(bearer)
+    if not match:
+        return False
+
+    tok = match.group(1)
+    if tok != settings.LOCALAPI_BEARER_TOKEN:
+        return False
+
+    return True
+
+def authenticated(func):
+    def wrapper(*args, **kwargs):
+        request=args[0]
+        if not authenticate_request(request):
+            return JsonResponse(dict(state="no :("), status=401)
+        return func(*args, **kwargs)
+    return wrapper
+#+end_src
+
+* Keyword metadata
+
+There is a simple set of HTTP GET APIs to query the file/key/value store:
+
+#+begin_src python :tangle localapi/views.py
+import roam.models
+
+def _json_keywords(keywords):
+    return [
+            dict(path=kw.path.path, keyword=kw.keyword, value=kw.value)
+            for kw in keywords
+        ]
+#+end_src
+
+** /keywords/{key}
+
+=GET http://127.0.0.1:8000/api/v1/keywords/ARCOLOGY_KEY=
+
+#+begin_src python :tangle localapi/views.py
+@authenticated
+def keyword_by_key(request, key):
+    keywords = roam.models.Keyword.objects.filter(keyword=key).all()
+    return JsonResponse(dict(
+        state="ok :)",
+        key=key,
+        keywords=_json_keywords(keywords),
+    ))
+#+end_src
+
+** /keywords/{key}/{value}
+
+=GET http://127.0.0.1:8000/api/v1/keywords/ARCOLOGY_KEY/arcology/localapi=
+
+#+begin_src python :tangle localapi/views.py
+@authenticated
+def keyword_by_key_value(request, key, value):
+    keywords = roam.models.Keyword.objects.filter(keyword=key, value=value).all()
+    return JsonResponse(dict(
+        state="ok :)",
+        key=key,
+        value=value,
+        keywords=_json_keywords(keywords),
+    ))
+#+end_src
+
+** NEXT add elisp APIs
+* Page and File metadata
+
+And some really simple APIs to get information about =Page= and =File= objects out of the database:
+
+#+begin_src python :tangle localapi/views.py
+import arcology.models
+
+def _json_page(page):
+    return dict(
+            title=page.title,
+            url=page.to_url(),
+            site=page.site.title,
+        )
+#+end_src
+
+=GET http://127.0.0.1:8000/api/v1/page/arcology/localapi= or this Emacs Lisp command:
+A command which will take an =ARCOLOGY_KEY= and the ID of a heading on that page, and return a URL for it:
+
+#+begin_src emacs-lisp :tangle ~/org/cce/arcology-localapi-commands.el
+(defun arcology-key-to-url (page-key &optional heading-id)
+  (interactive "sKey: \nsHeading ID: ")
+  (let ((url (concat
+              (thread-last
+                (arcology-localapi-call 'get (format  "/page/%s" page-key))
+                (alist-get 'page)
+                (alist-get 'url))
+              (when heading-id (format "#%s" heading-id)))))
+    (when (called-interactively-p)
+      (kill-new url)
+      (message "%s" url))
+    url))
+#+end_src
+
+#+begin_src python :tangle localapi/views.py
+@authenticated
+def page_metadata(request, route_key):
+    page = arcology.models.Page.objects.get(route_key=route_key)
+    keywords = page.file.keyword_set.all()
+    return JsonResponse(dict(
+        state="ok :)",
+        route_key=route_key,
+        file=page.file.path,
+        page=_json_page(page),
+        keywords=_json_keywords(keywords)
+    ))
+#+end_src
+
+=GET http://127.0.0.1:8000/api/v1/file/arcology-django/localapi.org= and the Emacs Lisp command which will do the same:
+
+#+begin_src emacs-lisp :tangle ~/org/cce/arcology-localapi-commands.el
+(defun arcology-file-to-url (file-path &optional heading-id)
+  (interactive "fKey: \nsHeading ID: ")
+  (let ((url (concat
+              (thread-last
+                (file-relative-name file-path org-roam-directory)
+                (format "http://127.0.0.1:8000/api/v1/file/%s")
+                (arcology-localapi-call 'get)
+                (alist-get 'file)
+                (alist-get 'page)
+                (alist-get 'url))
+              (when heading-id (format "#%s" heading-id)))))
+    (when (called-interactively-p)
+      (kill-new url)
+      (message "%s" url))
+    url))
+#+end_src
+
+#+begin_src python :tangle localapi/views.py
+from django.conf import settings
+
+@authenticated
+def file_metadata(request, file_path):
+    final_path = settings.ARCOLOGY_BASE_DIR.joinpath(file_path)
+    the_file = roam.models.File.objects.get(path=final_path)
+    page = the_file.page_set.first()
+    return JsonResponse(dict(
+        state="ok :)",
+        file_path=file_path,
+        file=dict(
+            path=the_file.path,
+            page=_json_page(page),
+        ),
+    ))
+#+end_src
+
+* Some more ELisp helpers
+:PROPERTIES:
+:ID:       20240313T212950.461285
+:END:
+
+#+begin_src emacs-lisp :tangle ~/org/cce/arcology-localapi-commands.el
+(defun arcology-url-at-point (pom)
+  (interactive "m")
+  (let ((url (arcology-key-to-url
+              (thread-last
+                (org-collect-keywords '("ARCOLOGY_KEY"))
+                (first)
+                (second))
+              (and (> (org-outline-level) 0)
+                   (org-id-get)))))
+    (when (called-interactively-p)
+      (kill-new url)
+      (message "%s" url))
+    url))
+#+end_src
+
+A command which will get the URL for a heading you can select using your =completing-read=:
+
+#+begin_src emacs-lisp :tangle ~/org/cce/arcology-localapi-commands.el
+(defun arcology-read-url ()
+  (interactive)
+  (let* ((node (org-roam-node-read))
+         (url (arcology-file-to-url (org-roam-node-file node)
+                                    (and (> (org-outline-level) 0)
+                                         (org-id-get)))))
+    (when (called-interactively-p)
+      (kill-new url)
+      (message "%s" url))))
+#+end_src
+
+* DONE [[id:20231217T154938.132553][Arcology generate Command]]
+:LOGBOOK:
+- State "DONE"       from "NEXT"       [2024-03-13 Wed 23:56]
+:END:
+
+emacs-lisp code along with the HTTP endpoints to populate things from the Generators
+
+#+begin_src emacs-lisp :tangle ~/org/cce/arcology-localapi-commands.el
+(defun arcology-api-generator (module &optional role destination do-sort)
+  (interactive)
+  (let ((url (format "/generate/%s%s" module
+                     (if role (format "/%s" role) ""))))
+    (let ((gen-text (with-current-buffer
+                        (arcology-localapi-call 'get url :as 'buffer)
+                      (buffer-substring-no-properties (point-min) (point-max))))
+          (buf (if destination
+                   (find-file destination)
+                 (get-buffer-create (format "*%s out*" module)))))
+      (with-current-buffer buf
+        (erase-buffer)
+        (insert gen-text)
+        (when do-sort
+          (sort-lines nil (point-min) (point-max)))
+        (when destination
+          (save-buffer))
+        (when (called-interactively-p)
+          (display-buffer))
+        (buffer-string)))))
+#+end_src
+
+#+begin_src emacs-lisp :tangle ~/org/cce/arcology-localapi-commands.el
+(provide 'cce/arcology-localapi-commands)
+#+end_src
+
+#+begin_src python :tangle localapi/views.py
+import tempfile
+from django.core.management import call_command
+
+@authenticated
+def generate(request, module, role=None):
+    f = tempfile.NamedTemporaryFile()
+    fn = f.name
+    # i hate this!!!
+    if role:
+        call_command('generate', destination=fn, module=module, role=role)
+    else:
+        call_command('generate', destination=fn, module=module)
+    f.seek(0)
+
+    return HttpResponse(f)
+#+end_src
+
+this is all full of jank. having to pass to a temporary file because the generate commands are locked away in a django management command is silly, i should refactor all of this. the branch is silly, too. But it's very easy now to get my literate programming helpers to quickly generate the code imports and whatnot as soon as the local server is running.
+
+* INPROGRESS [[id:cce/home-manager][home-manager]] deployment
+:LOGBOOK:
+- State "INPROGRESS" from "NEXT"       [2024-03-18 Mon 16:08]
+:END:
+
+This thing needs to run as a =systemd= User Unit for the local tangles and whatnot to work. Bootstrapping this will be Fun, we'll just end up needing a bootstrap script to put the =systemd= user units in place for future deployments...
+
+This is just the configurable module; the configuration itself is in the [[id:arcology/django/config][Arcology Project Configuration]] page like the Wobserver module.
+
+#+ARROYO_HOME_MODULE: hm/arcology-localapi.nix
+#+ARROYO_SYSTEM_ROLE: endpoint
+: #+ARROYO_SYSTEM_ROLE: droid # need to make sure there is a wrapper script to launch this
+
+#+begin_src nix :tangle ~/arroyo-nix/hm/arcology-localapi-mod.nix
+{ pkgs, config, lib, ... }:
+
+let
+  cfg = config.services.arcology2;
+  arcology = cfg.packages.arcology;
+
+  env = [
+    "ARCOLOGY_BASE_DIR=${cfg.orgDir}"
+    "ARCOLOGY_DB_PATH=${cfg.dataDir}/db.sqlite3"
+    "ARCOLOGY_LOG_LEVEL=${cfg.logLevel}"
+    "ARCOLOGY_CACHE_PATH=${cfg.cacheDir}"
+    "GUNICORN_CMD_ARGS='--bind=${cfg.address}:${toString cfg.port} -w ${toString cfg.workerCount}'"
+  ];
+
+  pyenv = pkgs.python3.withPackages(pp: [arcology]);
+
+  preStartScript = pkgs.writeScriptBin "arcology-watchsync-prestart" ''
+    mkdir -p ${cfg.dataDir}
+    ${arcology}/bin/arcology migrate
+    ${arcology}/bin/arcology seed || true
+  '';
+  watchsyncScript = pkgs.writeScriptBin "arcology-watchsync" ''
+    ${arcology}/bin/arcology watchsync -f ${cfg.folderId}
+  '';
+  localapiScript = pkgs.writeScriptBin "arcology-localapi" ''
+    ${pyenv}/bin/python -m gunicorn arcology.wsgi
+  '';
+in {
+  options.services.arcology2 = with lib; {
+    enable = mkEnableOption (mdDoc "Arcology Local API");
+
+    packages.arcology = mkOption {
+      type = types.package;
+      description = mdDoc ''
+      '';
+    };
+
+    address = mkOption {
+      type = types.str;
+      default = "localhost";
+      description = lib.mdDoc "Web interface address.";
+    };
+
+    port = mkOption {
+      type = types.port;
+      default = 29543;
+      description = lib.mdDoc "Web interface port.";
+    };
+
+    workerCount = mkOption {
+      type = types.number;
+      default = 4;
+      description = lib.mdDoc "gunicorn worker count; they recommend 2-4 workers per core.";
+    };
+
+    logLevel = mkOption {
+      type = types.enum ["ERROR" "WARN" "INFO" "DEBUG"];
+      default = "INFO";
+      description = mdDoc ''
+        Set the Django root logging level
+      '';
+    };
+
+    orgDir = mkOption {
+      type = types.str;
+      default = "~/org";
+      description = mdDoc ''
+        Directory containing the org-mode documents.
+        Arcology needs read-only access to this directory.
+      '';
+    };
+
+    environmentFile = mkOption {
+      type = types.path;
+      default = "${cfg.dataDir}/env";
+      description = mdDoc ''
+        A file containing environment variables you may not want to put in the nix store.
+
+        For example, you could put a syncthing key 
+        and a bearer token for the Local API in there:
+
+        ARCOLOGY_SYNCTHING_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA;
+        ARCOLOGY_LOCALAPI_BEARER_TOKEN=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA;
+      '';
+    };
+
+    folderId = mkOption {
+      type = types.str;
+      description = mdDoc ''
+        Syncthing folder ID containing the org files.
+      '';
+    };
+
+    cacheDir = mkOption {
+      type = types.str;
+      default = "~/.cache/arcology2/";
+      description = mdDoc ''
+        Location to cache HTML files and the like.
+      '';
+    };
+
+    dataDir = mkOption {
+      type = types.str;
+      default = "~/.config/arcology2/";
+      description = mdDoc ''
+        Location to store the arcology metadata store.
+      '';
+    };
+  };
+
+  config = {
+    systemd.user.services = {
+      arcology-watchsync = {
+        Unit.Description = "Dynamically update the Arcology database.";
+        Service = {
+          ExecStartPre = "${pkgs.stdenv.shell} ${preStartScript}/bin/arcology-watchsync-prestart";
+          ExecStart = "${pkgs.stdenv.shell} ${watchsyncScript}/bin/arcology-watchsync";
+
+          Restart = "on-failure";
+          Environment = env;
+          EnvironmentFile = cfg.environmentFile;
+        };
+        Install.WantedBy = ["default.target"];
+      };
+
+      arcology-web = {
+        Unit.Description = "Local hosted version of the Arcology";
+        Service = {
+          ExecStart = "${pkgs.stdenv.shell} ${localapiScript}/bin/arcology-localapi";
+
+          Restart = "on-failure";
+          Environment = env;
+          EnvironmentFile = cfg.environmentFile;
+        };
+        Install.WantedBy = ["default.target"];
+      };
+    };
+  };
+}
+#+end_src
+
+** NEXT DRY this between here and the [[id:20240213T124300.774781][Deploying the Arcology]] work.
+** NEXT is the arcology wobserver deployment better as a systemd user unit?
+
+still need nixos module for the vhosts etc, but... hm.

localapi/auth.py

@@ -0,0 +1,24 @@
+import re
+from django.conf import settings
+from django.http import HttpRequest, JsonResponse
+
+def authenticate_request(request: HttpRequest):
+    r = re.compile(r'Bearer (\S+)')
+    bearer = request.headers.get("Authorization", "")
+    match = r.match(bearer)
+    if not match:
+        return False
+
+    tok = match.group(1)
+    if tok != settings.LOCALAPI_BEARER_TOKEN:
+        return False
+
+    return True
+
+def authenticated(func):
+    def wrapper(*args, **kwargs):
+        request=args[0]
+        if not authenticate_request(request):
+            return JsonResponse(dict(state="no :("), status=401)
+        return func(*args, **kwargs)
+    return wrapper

localapi/urls.py

@@ -0,0 +1,14 @@
+from django.contrib import admin
+from django.urls import path, re_path, include
+
+from localapi import views
+
+urlpatterns = [
+    path("", views.index),
+    path("generate/<slug:module>/<slug:role>", views.generate, name="keyword_by_key"),
+    path("generate/<slug:module>", views.generate, name="keyword_by_key"),
+    path("keywords/<slug:key>", views.keyword_by_key, name="keyword_by_key"),
+    re_path("keywords/(?P<key>[0-9a-zA-Z_-]+)/(?P<value>[0-9a-zA-Z/_\-]+)", views.keyword_by_key_value, name="keyword_by_key"),
+    re_path("page/(?P<route_key>[0-9a-zA-Z/_\-]+)", views.page_metadata, name="page_metadata"),
+    re_path("file/(?P<file_path>[0-9a-zA-Z/_\-\.]+)", views.file_metadata, name="file_metadata"),
+]

localapi/views.py

@@ -0,0 +1,95 @@
+from django.http import HttpRequest, HttpResponse, HttpResponseNotFound, Http404, JsonResponse
+from django.shortcuts import render, get_object_or_404
+
+from arcology.models import Page, Feed, Site
+from roam.models import Link
+from localapi.auth import authenticated
+
+from prometheus_client import Counter, Histogram
+
+import logging
+logger = logging.getLogger(__name__)
+
+@authenticated
+def index(request):
+    return JsonResponse(dict(state="ok :)"))
+
+import roam.models
+
+def _json_keywords(keywords):
+    return [
+            dict(path=kw.path.path, keyword=kw.keyword, value=kw.value)
+            for kw in keywords
+        ]
+
+@authenticated
+def keyword_by_key(request, key):
+    keywords = roam.models.Keyword.objects.filter(keyword=key).all()
+    return JsonResponse(dict(
+        state="ok :)",
+        key=key,
+        keywords=_json_keywords(keywords),
+    ))
+
+@authenticated
+def keyword_by_key_value(request, key, value):
+    keywords = roam.models.Keyword.objects.filter(keyword=key, value=value).all()
+    return JsonResponse(dict(
+        state="ok :)",
+        key=key,
+        value=value,
+        keywords=_json_keywords(keywords),
+    ))
+
+import arcology.models
+
+def _json_page(page):
+    return dict(
+            title=page.title,
+            url=page.to_url(),
+            site=page.site.title,
+        )
+
+@authenticated
+def page_metadata(request, route_key):
+    page = arcology.models.Page.objects.get(route_key=route_key)
+    keywords = page.file.keyword_set.all()
+    return JsonResponse(dict(
+        state="ok :)",
+        route_key=route_key,
+        file=page.file.path,
+        page=_json_page(page),
+        keywords=_json_keywords(keywords)
+    ))
+
+from django.conf import settings
+
+@authenticated
+def file_metadata(request, file_path):
+    final_path = settings.ARCOLOGY_BASE_DIR.joinpath(file_path)
+    the_file = roam.models.File.objects.get(path=final_path)
+    page = the_file.page_set.first()
+    return JsonResponse(dict(
+        state="ok :)",
+        file_path=file_path,
+        file=dict(
+            path=the_file.path,
+            page=_json_page(page),
+        ),
+    ))
+
+import tempfile
+from django.core.management import call_command
+
+@authenticated
+def generate(request, module, role=None):
+    f = tempfile.NamedTemporaryFile()
+    fn = f.name
+    # i hate this!!!
+    if role:
+        call_command('generate', destination=fn, module=module, role=role)
+    else:
+        call_command('generate', destination=fn, module=module)
+    f.seek(0)
+
+    return HttpResponse(f)

scaffolding.org

@@ -201,6 +201,7 @@ Nix is really going this direction, I'm not sure it's worthwhile but I'm going t
 }
 #+end_src
 
+*** NEXT expose nixos modules and home manager modules here to aid re-bootstrap
 ** Direnv
 
 [[id:45fc2a02-fcd0-40c6-a29e-897c0ee7b1c7][direnv]] fucking rules.

syncthonk/management/commands/watchsync.py

@@ -5,14 +5,16 @@ from django.conf import settings
 
 import json
 import pathlib
-import urllib.parse 
-import urllib.request 
+import urllib.parse
+import urllib.request
 
 import polling
 
 import logging
 logger = logging.getLogger(__name__)
 # logger.setLevel(logging.DEBUG)
+
+import roam.models
 # Ingest files on-demand using Syncthing:2 ends here
 
 # [[file:../../../interfaces.org::*Ingest files on-demand using Syncthing][Ingest files on-demand using Syncthing:3]]
@@ -88,7 +90,8 @@ class Command(BaseCommand):
         last_since = None
         for event in jason:
             last_since = event.get("id")
-            event_folder_id = event.get("data", {}).get("folder", "")
+            data = event.get("data", {})
+            event_folder_id = data.get("folder", "")
             file_path = pathlib.Path(event.get("data", {}).get("path"))
 # Ingest files on-demand using Syncthing:6 ends here
 
@@ -103,12 +106,27 @@ class Command(BaseCommand):
             elif event_folder_id != self.folder_id:
                 logger.debug(f"skip unmonitored folder {event_folder_id}")
                 ingest_this = False
+# Ingest files on-demand using Syncthing:7 ends here
+
+# [[file:../../../interfaces.org::*Ingest files on-demand using Syncthing][Ingest files on-demand using Syncthing:8]]
+            elif data.get("action") == "deleted":
+                final_path = self.expanded_path.joinpath(file_path)
+                try:
+                    f = roam.models.File.objects.get(path=final_path)
+                    logger.debug(f"deleting {final_path}!")
+                    f.delete()
+                    ingest_this = False
+                except roam.models.File.DoesNotExist:
+                    pass
+# Ingest files on-demand using Syncthing:8 ends here
+
+# [[file:../../../interfaces.org::*Ingest files on-demand using Syncthing][Ingest files on-demand using Syncthing:9]]
             # add new failure cases here.
-            logger.debug(f"proc {last_since} {event_folder_id}, ingest? {ingest}, ingest_this? {ingest_this}")
+            logger.debug(f"proc {last_since} {event_folder_id}, ingest? {ingest}, ingest_this? {ingest_this}: {json.dumps(event)}")
             if ingest_this == True:
                 ingest = ingest_this
                 logger.debug(f"{event}")
         if ingest:
           call_command('ingestfiles', self.expanded_path)
         return last_since
-# Ingest files on-demand using Syncthing:7 ends here
+# Ingest files on-demand using Syncthing:9 ends here