add basic metadata views to [localapi] and authentication decorator

arcology.org

@@ -540,6 +540,7 @@ urlpatterns = [
     path("feeds.json", views.feed_list, name="feed-list"),
     path("", include("django_prometheus.urls")),
     path("", include("sitemap.urls")),
+    path("api/v1/", include("localapi.urls")),
     # ensure these ones are last because they're greedy!
     re_path("(?P<key>[0-9a-zA-Z/_\-]+\.xml)", views.feed, name="feed"),
     re_path("(?P<key>[0-9a-zA-Z/_\-]+)", views.org_page, name="org-page"),

arcology/settings/__init__.py

@@ -32,6 +32,10 @@ CACHES = {
 }
 # Environment Variables:4 ends here
 
+# [[file:../../configuration.org::*Environment Variables][Environment Variables:5]]
+LOCALAPI_BEARER_TOKEN = os.environ.get("ARCOLOGY_LOCALAPI_BEARER_TOKEN", "changeme!")
+# Environment Variables:5 ends here
+
 # [[file:../../configuration.org::*Hostname configuration from =arcology.model.Site=, eventually][Hostname configuration from =arcology.model.Site=, eventually:1]]
 ALLOWED_HOSTS = "thelionsrear.com,rix.si,arcology.garden,whatthefuck.computer,cce.whatthefuck.computer,cce.rix.si,engine.arcology.garden,127.0.0.1,localhost,v2.thelionsrear.com,v2.arcology.garden,cce2.whatthefuck.computer,engine2.arcology.garden".split(',')
 # Hostname configuration from =arcology.model.Site=, eventually:1 ends here
@@ -44,6 +48,7 @@ INSTALLED_APPS = [
     "generators", # [[id:arroyo/django/generators][The Arroyo Generators]]
     "syncthonk", # [[id:20231218T183551.765340][Arcology watchsync Command]]
     "sitemap", # [[id:20240226T132507.817450][The Arcology's Site Maps and Discovery Mechanisms]]
+    "localapi", # [[id:20240313T153901.656967][A Localhost API for the Arcology]]
     
     "django_htmx",
     "django_prometheus",

arcology/urls.py

@@ -13,6 +13,7 @@ urlpatterns = [
     path("feeds.json", views.feed_list, name="feed-list"),
     path("", include("django_prometheus.urls")),
     path("", include("sitemap.urls")),
+    path("api/v1/", include("localapi.urls")),
     # ensure these ones are last because they're greedy!
     re_path("(?P<key>[0-9a-zA-Z/_\-]+\.xml)", views.feed, name="feed"),
     re_path("(?P<key>[0-9a-zA-Z/_\-]+)", views.org_page, name="org-page"),

configuration.org

@@ -240,6 +240,10 @@ CACHES = {
 }
 #+END_SRC
 
+#+begin_src python :tangle arcology/settings/__init__.py
+LOCALAPI_BEARER_TOKEN = os.environ.get("ARCOLOGY_LOCALAPI_BEARER_TOKEN", "changeme!")
+#+end_src
+
 ** NEXT Hostname configuration from =arcology.model.Site=, eventually
 
 When I have the sites organized in an org-mode table, i'll reapproach the hostname list, and probably before then when i want to test domain-based routing.
@@ -258,6 +262,7 @@ basically, each org file in this repository, and maybe one or two of your own, a
 | "generators" | [[id:arroyo/django/generators][The Arroyo Generators]]                             |
 | "syncthonk"  | [[id:20231218T183551.765340][Arcology watchsync Command]]                        |
 | "sitemap"    | [[id:20240226T132507.817450][The Arcology's Site Maps and Discovery Mechanisms]] |
+| "localapi"   | [[id:20240313T153901.656967][A Localhost API for the Arcology]]                  |
 
 #+BEGIN_SRC python :tangle arcology/settings/__init__.py :noweb yes
 # Application definition

deployment.org

@@ -206,9 +206,11 @@ in {
         description = mdDoc ''
           A file containing environment variables you may not want to put in the nix store.
 
-          For example, you could put a syncthing key in there:
+          For example, you could put a syncthing key 
+          and a bearer token for the Local API in there:
 
           ARCOLOGY_SYNCTHING_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA;
+          ARCOLOGY_LOCALAPI_BEARER_TOKEN=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA;
         '';
       };
 

interfaces.org

@@ -1,4 +1,5 @@
 # -*- org-src-preserve-indentation: t; -*-
+#+filetags: :Project:
 :PROPERTIES:
 :ID:       arcology/django/interfaces
 :ROAM_ALIASES: "The Arcology Management Commands"
@@ -279,14 +280,16 @@ from django.conf import settings
 
 import json
 import pathlib
-import urllib.parse 
+import urllib.parse
-import urllib.request 
+import urllib.request
 
 import polling
 
 import logging
 logger = logging.getLogger(__name__)
 # logger.setLevel(logging.DEBUG)
+
+import roam.models
 #+end_src
 
 The command goes in to a simple loop with [[https://github.com/justiniso/polling][=polling=]] used to ensure that transient errors are retried and the loop does not die without good cause. The [[id:cce/syncthing][Syncthing]] API key is loaded from the [[id:arcology/django/config][Project Configuration]] and the path to monitor is passed in as an argument. Note that this requires the path to match a Syncthing directory! If you are monitoring a folder which is subdirectory of a shared folder, make sure to pass the root of the shared folder in instead or break the path in to its own Syncthing share.
@@ -315,7 +318,7 @@ class Command(BaseCommand):
             self.folder_id = self.get_folder_id_for_path(self.expanded_path)
             logger.info(f"fetched folder ID {self.folder_id} AKA {self.expanded_path} from syncthing API")
         #+end_src
-        
+
 it would be nice to get the pagination key from the DB to have persistence across invocations, but re-running the ingester on startup when there isn't necessarily changes isn't really that bad, it'll just cause some disk IO while it checks file hashes:
 
 #+begin_src python :tangle syncthonk/management/commands/watchsync.py
@@ -372,7 +375,8 @@ The functionality to query the Syncthing [[https://docs.syncthing.net/rest/event
         last_since = None
         for event in jason:
             last_since = event.get("id")
-            event_folder_id = event.get("data", {}).get("folder", "")
+            data = event.get("data", {})
+            event_folder_id = data.get("folder", "")
             file_path = pathlib.Path(event.get("data", {}).get("path"))
 #+end_src
 
@@ -389,8 +393,23 @@ For each file path, it checks that the file name is not a temporary file name, t
             elif event_folder_id != self.folder_id:
                 logger.debug(f"skip unmonitored folder {event_folder_id}")
                 ingest_this = False
+            #+end_src
+
+If a file is deleted, make sure it's removed from the database. Note that this is still liable to miss files; eventually I will want to add a management command to go over the whole file listing and true up the database, but this is good enough for now as long as it's kept to be realtime.
+
+            #+begin_src python :tangle syncthonk/management/commands/watchsync.py
+            elif data.get("action") == "deleted":
+                final_path = self.expanded_path.joinpath(file_path)
+                f = roam.models.File.objects.get(path=final_path)
+                assert f
+                logger.debug(f"deleting {final_path}!")
+                f.delete()
+                ingest_this = False
+            #+end_src
+
+            #+begin_src python :tangle syncthonk/management/commands/watchsync.py
             # add new failure cases here.
-            logger.debug(f"proc {last_since} {event_folder_id}, ingest? {ingest}, ingest_this? {ingest_this}")
+            logger.debug(f"proc {last_since} {event_folder_id}, ingest? {ingest}, ingest_this? {ingest_this}: {json.dumps(event)}")
             if ingest_this == True:
                 ingest = ingest_this
                 logger.debug(f"{event}")
@@ -401,6 +420,11 @@ For each file path, it checks that the file name is not a temporary file name, t
 
 The command returns the last event ID of the request which is used to paginate future requests made in the loop.
 
+** DONE make sure we are able to handle deletions here...
+:LOGBOOK:
+- State "DONE"       from "NEXT"       [2024-03-13 Wed 15:37]
+:END:
+
 * Create Base Sites
 :PROPERTIES:
 :ID:       20231217T154835.232283

localapi.org

@@ -0,0 +1,171 @@
+:PROPERTIES:
+:ID:       20240313T153901.656967
+:END:
+#+TITLE: A Localhost API for the Arcology
+#+filetags: :Project:
+
+By moving the database out of EmacSQL and in to Django, I *have* hobbled some of the Arcology's [[id:knowledge_base][Knowledge Management]] [[id:e79d4cdc-082f-4b1d-ae08-d979802f09ee][Living Systems]] and meta-cognitive skills. =arroyo-db-query= no longer existing means that things like the [[id:20230526T105534.711282][Direnv arroyo-db integration]] stopped working, and the helpers in [[id:cce/my_nixos_configuration][My NixOS configuration]] and elsewhere rely on the stringly-typed [[id:20231217T154938.132553][Arcology generate Command]] and a =nix run= invocation.
+
+I think it's worth building a small HTTP JSON API which can serve some of the common queries, to at least see if it's worth the trouble.
+
+I probably should use Django REST Framework for this, but. I'm just gonna beat two stones together until a JSON API falls out. This will probably change the first time I add a =POST= call or want to do more complex auth stuff.
+
+* Server view scaffolding
+
+#+begin_src python :tangle localapi/urls.py :mkdirp yes
+from django.contrib import admin
+from django.urls import path, re_path, include
+
+from localapi import views
+
+urlpatterns = [
+    path("", views.index),
+    path("keywords/<slug:key>", views.keyword_by_key, name="keyword_by_key"),
+    re_path("keywords/(?P<key>[0-9a-zA-Z_-]+)/(?P<value>[0-9a-zA-Z/_\-]+)", views.keyword_by_key_value, name="keyword_by_key"),
+    path("page/<>", views.page_metadata, name="page_metadata"),
+    re_path("page/(?P<route_key>[0-9a-zA-Z/_\-]+)", views.page_metadata, name="page_metadata"),
+    re_path("file/(?P<file_path>[0-9a-zA-Z/_\-\.]+)", views.file_metadata, name="file_metadata"),
+]
+#+end_src
+
+#+begin_src python :tangle localapi/views.py
+from django.http import HttpRequest, HttpResponse, HttpResponseNotFound, Http404, JsonResponse
+from django.shortcuts import render, get_object_or_404
+
+from arcology.models import Page, Feed, Site
+from roam.models import Link
+from localapi.auth import authenticated
+
+from prometheus_client import Counter, Histogram
+
+import logging
+logger = logging.getLogger(__name__)
+#+end_src
+
+The API is fairly simple and built-to-purpose, we start with an endpoint that can be queried to validate that the API Bearer token is valid:
+
+#+begin_src python :tangle localapi/views.py
+@authenticated
+def index(request):
+    return JsonResponse(dict(state="ok :)"))
+#+end_src
+
+* DONE Auth Middleware
+:LOGBOOK:
+- State "DONE"       from              [2024-03-13 Wed 17:01]
+:END:
+
+The Bearer token is set in the [[id:arcology/django/config][Arcology Project Configuration]], and can be overridden in an environment variable =ARCOLOGY_LOCALAPI_BEARER_TOKEN=. The NixOS endpoint configuration module for this API will include token generation and placement so that this should be transparent to a local command user. Decorating a view with =@authenticated= will add this bearer token authentication to the view.
+
+#+begin_src python :tangle localapi/auth.py
+import re
+from django.conf import settings
+from django.http import HttpRequest, JsonResponse
+
+def authenticate_request(request: HttpRequest):
+    r = re.compile(r'Bearer (\S+)')
+    bearer = request.headers.get("Authorization", "")
+    match = r.match(bearer)
+    if not match:
+        return False
+
+    tok = match.group(1)
+    if tok != settings.LOCALAPI_BEARER_TOKEN:
+        return False
+
+    return True
+
+def authenticated(func):
+    def wrapper(*args, **kwargs):
+        request=args[0]
+        if not authenticate_request(request):
+            return JsonResponse(dict(state="no :("), status=401)
+        return func(*args, **kwargs)
+    return wrapper
+#+end_src
+
+* Keyword metadata
+
+#+begin_src python :tangle localapi/views.py
+import roam.models
+
+def _json_keywords(keywords):
+    return [
+            dict(path=kw.path.path, keyword=kw.keyword, value=kw.value)
+            for kw in keywords
+        ]
+#+end_src
+
+** /keywords/{key}
+
+#+begin_src python :tangle localapi/views.py
+@authenticated
+def keyword_by_key(request, key):
+    keywords = roam.models.Keyword.objects.filter(keyword=key).all()
+    return JsonResponse(dict(
+        state="ok :)",
+        key=key,
+        keywords=_json_keywords(keywords),
+    ))
+#+end_src
+
+** /keywords/{key}/{value}
+
+#+begin_src python :tangle localapi/views.py
+@authenticated
+def keyword_by_key_value(request, key, value):
+    keywords = roam.models.Keyword.objects.filter(keyword=key, value=value).all()
+    return JsonResponse(dict(
+        state="ok :)",
+        key=key,
+        value=value,
+        keywords=_json_keywords(keywords),
+    ))
+#+end_src
+
+* Page and File metadata
+
+#+begin_src python :tangle localapi/views.py
+import arcology.models
+
+def _json_page(page):
+    return dict(
+            title=page.title,
+            url=page.to_url(),
+            site=page.site.title,
+        )
+#+end_src
+
+#+begin_src python :tangle localapi/views.py
+@authenticated
+def page_metadata(request, route_key):
+    page = arcology.models.Page.objects.get(route_key=route_key)
+    keywords = page.file.keyword_set.all()
+    return JsonResponse(dict(
+        state="ok :)",
+        route_key=route_key,
+        file=page.file.path,
+        page=_json_page(page),
+        keywords=_json_keywords(keywords)
+    ))
+#+end_src
+
+#+begin_src python :tangle localapi/views.py
+@authenticated
+def file_metadata(request, file_path):
+    the_file = roam.models.File.objects.get(path=file_path)
+    page = the_file.page_set.first()
+    return JsonResponse(dict(
+        state="ok :)",
+        file_path=file_path,
+        file=dict(
+            path=the_file.path,
+            page=_json_page(page),
+        ),
+    ))
+#+end_src
+
+* NEXT [[id:20221021T150631.404359][Arroyo Nix Library Helpers]]
+* NEXT [[id:20231217T154938.132553][Arcology generate Command]]
+* NEXT NixOS deployment for endpoints in [[id:cce/my_nixos_configuration][My NixOS configuration]]
+

localapi/auth.py

@@ -0,0 +1,26 @@
+# [[file:../localapi.org::*Auth Middleware][Auth Middleware:1]]
+import re
+from django.conf import settings
+from django.http import HttpRequest, JsonResponse
+
+def authenticate_request(request: HttpRequest):
+    r = re.compile(r'Bearer (\S+)')
+    bearer = request.headers.get("Authorization", "")
+    match = r.match(bearer)
+    if not match:
+        return False
+
+    tok = match.group(1)
+    if tok != settings.LOCALAPI_BEARER_TOKEN:
+        return False
+
+    return True
+
+def authenticated(func):
+    def wrapper(*args, **kwargs):
+        request=args[0]
+        if not authenticate_request(request):
+            return JsonResponse(dict(state="no :("), status=401)
+        return func(*args, **kwargs)
+    return wrapper
+# Auth Middleware:1 ends here

localapi/urls.py

@@ -0,0 +1,15 @@
+# [[file:../localapi.org::*Server view scaffolding][Server view scaffolding:1]]
+from django.contrib import admin
+from django.urls import path, re_path, include
+
+from localapi import views
+
+urlpatterns = [
+    path("", views.index),
+    path("keywords/<slug:key>", views.keyword_by_key, name="keyword_by_key"),
+    re_path("keywords/(?P<key>[0-9a-zA-Z_-]+)/(?P<value>[0-9a-zA-Z/_\-]+)", views.keyword_by_key_value, name="keyword_by_key"),
+    path("page/<>", views.page_metadata, name="page_metadata"),
+    re_path("page/(?P<route_key>[0-9a-zA-Z/_\-]+)", views.page_metadata, name="page_metadata"),
+    re_path("file/(?P<file_path>[0-9a-zA-Z/_\-\.]+)", views.file_metadata, name="file_metadata"),
+]
+# Server view scaffolding:1 ends here

localapi/views.py

@@ -0,0 +1,92 @@
+# [[file:../localapi.org::*Server view scaffolding][Server view scaffolding:2]]
+from django.http import HttpRequest, HttpResponse, HttpResponseNotFound, Http404, JsonResponse
+from django.shortcuts import render, get_object_or_404
+
+from arcology.models import Page, Feed, Site
+from roam.models import Link
+from localapi.auth import authenticated
+
+from prometheus_client import Counter, Histogram
+
+import logging
+logger = logging.getLogger(__name__)
+# Server view scaffolding:2 ends here
+
+# [[file:../localapi.org::*Server view scaffolding][Server view scaffolding:3]]
+@authenticated
+def index(request):
+    return JsonResponse(dict(state="ok :)"))
+# Server view scaffolding:3 ends here
+
+# [[file:../localapi.org::*Keyword metadata][Keyword metadata:1]]
+import roam.models
+
+def _json_keywords(keywords):
+    return [
+            dict(path=kw.path.path, keyword=kw.keyword, value=kw.value)
+            for kw in keywords
+        ]
+# Keyword metadata:1 ends here
+
+# [[file:../localapi.org::*/keywords/{key}][/keywords/{key}:1]]
+@authenticated
+def keyword_by_key(request, key):
+    keywords = roam.models.Keyword.objects.filter(keyword=key).all()
+    return JsonResponse(dict(
+        state="ok :)",
+        key=key,
+        keywords=_json_keywords(keywords),
+    ))
+# /keywords/{key}:1 ends here
+
+# [[file:../localapi.org::*/keywords/{key}/{value}][/keywords/{key}/{value}:1]]
+@authenticated
+def keyword_by_key_value(request, key, value):
+    keywords = roam.models.Keyword.objects.filter(keyword=key, value=value).all()
+    return JsonResponse(dict(
+        state="ok :)",
+        key=key,
+        value=value,
+        keywords=_json_keywords(keywords),
+    ))
+# /keywords/{key}/{value}:1 ends here
+
+# [[file:../localapi.org::*Page and File metadata][Page and File metadata:1]]
+import arcology.models
+
+def _json_page(page):
+    return dict(
+            title=page.title,
+            url=page.to_url(),
+            site=page.site.title,
+        )
+# Page and File metadata:1 ends here
+
+# [[file:../localapi.org::*Page and File metadata][Page and File metadata:2]]
+@authenticated
+def page_metadata(request, route_key):
+    page = arcology.models.Page.objects.get(route_key=route_key)
+    keywords = page.file.keyword_set.all()
+    return JsonResponse(dict(
+        state="ok :)",
+        route_key=route_key,
+        file=page.file.path,
+        page=_json_page(page),
+        keywords=_json_keywords(keywords)
+    ))
+# Page and File metadata:2 ends here
+
+# [[file:../localapi.org::*Page and File metadata][Page and File metadata:3]]
+@authenticated
+def file_metadata(request, file_path):
+    the_file = roam.models.File.objects.get(path=file_path)
+    page = the_file.page_set.first()
+    return JsonResponse(dict(
+        state="ok :)",
+        file_path=file_path,
+        file=dict(
+            path=the_file.path,
+            page=_json_page(page),
+        ),
+    ))
+# Page and File metadata:3 ends here

syncthonk/management/commands/watchsync.py

@@ -5,14 +5,16 @@ from django.conf import settings
 
 import json
 import pathlib
-import urllib.parse 
+import urllib.parse
-import urllib.request 
+import urllib.request
 
 import polling
 
 import logging
 logger = logging.getLogger(__name__)
 # logger.setLevel(logging.DEBUG)
+
+import roam.models
 # Ingest files on-demand using Syncthing:2 ends here
 
 # [[file:../../../interfaces.org::*Ingest files on-demand using Syncthing][Ingest files on-demand using Syncthing:3]]
@@ -88,7 +90,8 @@ class Command(BaseCommand):
         last_since = None
         for event in jason:
             last_since = event.get("id")
-            event_folder_id = event.get("data", {}).get("folder", "")
+            data = event.get("data", {})
+            event_folder_id = data.get("folder", "")
             file_path = pathlib.Path(event.get("data", {}).get("path"))
 # Ingest files on-demand using Syncthing:6 ends here
 
@@ -103,12 +106,25 @@ class Command(BaseCommand):
             elif event_folder_id != self.folder_id:
                 logger.debug(f"skip unmonitored folder {event_folder_id}")
                 ingest_this = False
+# Ingest files on-demand using Syncthing:7 ends here
+
+# [[file:../../../interfaces.org::*Ingest files on-demand using Syncthing][Ingest files on-demand using Syncthing:8]]
+            elif data.get("action") == "deleted":
+                final_path = self.expanded_path.joinpath(file_path)
+                f = roam.models.File.objects.get(path=final_path)
+                assert f
+                logger.debug(f"deleting {final_path}!")
+                f.delete()
+                ingest_this = False
+# Ingest files on-demand using Syncthing:8 ends here
+
+# [[file:../../../interfaces.org::*Ingest files on-demand using Syncthing][Ingest files on-demand using Syncthing:9]]
             # add new failure cases here.
-            logger.debug(f"proc {last_since} {event_folder_id}, ingest? {ingest}, ingest_this? {ingest_this}")
+            logger.debug(f"proc {last_since} {event_folder_id}, ingest? {ingest}, ingest_this? {ingest_this}: {json.dumps(event)}")
             if ingest_this == True:
                 ingest = ingest_this
                 logger.debug(f"{event}")
         if ingest:
           call_command('ingestfiles', self.expanded_path)
         return last_since
-# Ingest files on-demand using Syncthing:7 ends here
+# Ingest files on-demand using Syncthing:9 ends here