getting nixos module to work, doesn't quite boot yet

README.org

@@ -2,7 +2,7 @@
 :ID:       arcology/django/readme
 :END:
 #+TITLE: The Arcology Project: Django Edition
-#+FILETAGS: :README:
+#+FILETAGS: :Project:README:
 #+ARCOLOGY_KEY: arcology/django
 
 * The Arcology Project
@@ -40,7 +40,8 @@ ofc i need to flesh all this out still. don't use this, for now.
 This project is a living document. It's [[id:cce/literate_programming][Literate Programming]] powered by [[id:cce/literate_programming][Org Babel]]. You shouldn't need to read all the code to run it, but if you need to it oughta be well documented.
 
 - [[id:arcology/django/config][Arcology Project configuration]] describes and implements the knobs the user can twiddle to configure the Arcology's operation
-- [[id:arcology/django/interfaces][Interfacing with the Arcology]] is a set of management commands and a Syncthing client which will automatically ingest new files in to the database. This document describes all the tools a prospective user of the CCE will need to use for surface-level features.
+- [[id:arcology/django/interfaces][Interfacing with the Arcology]] provides a set of management commands and a Syncthing client which will automatically ingest new files in to the database. This document describes all the tools a prospective user of the CCE will need to use for surface-level features.
+- [[id:20240213T124300.774781][Deploying the Arcology]] lays out the basics of operating a Wobserver, but also provides a methodology for bootstrapping the Arcology on systems not already running NixOS.
 - The [[id:arcology/django/scaffolding][Arcology Project Scaffolding]] contains the files necessary to run the project, defines the nix environment, and the python environment, and the base Django apparatus.
 - The [[id:arcology/django/roam][Arcology Roam Models]] are the base metadata of the [[id:cce/org-roam][org-roam]] documents
 - [[id:arcology/django/arcology-models][The Arcology's Data Models and Web Server]] are the web publishing platform and the data models it requires to be operated
@@ -51,12 +52,25 @@ This project is a living document. It's [[id:cce/literate_programming][Literate
 :ID:       20240205T101753.548048
 :END:
 
-- tests, unit tests, functional tests with org-documents, eugh.
-- Prometheus and basic page hit count/analytics
-- Deployment and NixOS module to testing-subdomains
-- Sitemap
-- Move [[id:arroyo/django/generators][The Arroyo Generators]] and perhaps [[id:arcology/django/roam][The Arcology Roam Models]] in to [[id:arroyo/arroyo][Arroyo Systems Management]]
-- [[id:20231113T195508.942155][Rebuild of The Complete Computer]]
-- Consider having a second sqlite3 with server-written state like hit counts and fedi urls and whatnot that i am nervous to store in a DB right now.
+** INPROGRESS tests, unit tests, functional tests with org-documents, eugh.
+:LOGBOOK:
+- State "INPROGRESS" from "NEXT"       [2024-02-12 Mon 12:10]
+:END:
+
+** INPROGRESS Prometheus and basic page hit count/analytics
+:LOGBOOK:
+- State "INPROGRESS" from "NEXT"       [2024-02-12 Mon 12:10]
+:END:
+
+** DONE robots.txt
+:LOGBOOK:
+- State "DONE"       from "NEXT"       [2024-02-12 Mon 12:10]
+:END:
+
+** NEXT Deployment and NixOS module to testing-subdomains
+** NEXT Sitemap
+** NEXT Move [[id:arroyo/django/generators][The Arroyo Generators]] and perhaps [[id:arcology/django/roam][The Arcology Roam Models]] in to [[id:arroyo/arroyo][Arroyo Systems Management]]
+** NEXT [[id:20231113T195508.942155][Rebuild of The Complete Computer]]
+** NEXT Consider having a second sqlite3 with server-written state like hit counts and fedi urls and whatnot that i am nervous to store in a DB right now.
 
 * [[id:20220116T143655.499306][Hey Smell This]]

arcology.org

@@ -540,12 +540,14 @@ urlpatterns = [
 
 #+begin_src python :tangle arcology/views.py
 import logging
-from django.http import HttpResponse, HttpResponseNotFound
+from django.http import HttpResponse, HttpResponseNotFound, Http404
 from django.shortcuts import render, get_object_or_404
 
 from arcology.models import Page, Feed, Site
 from roam.models import Link
 
+from prometheus_client import Counter, Histogram
+
 logger = logging.getLogger(__name__)
 #+end_src
 
@@ -556,11 +558,9 @@ this will just call the org-page rendering handler for the site's index pages ev
 #+begin_src python :tangle arcology/views.py
 def index(request):
     site = Site.from_request(request)
-    logger.warn(site)
 
     full_key = f"{site.key}/index"
 
-    logger.warn(full_key)
     return render_page(request, site, full_key)
 #+end_src
 
@@ -584,30 +584,47 @@ def org_page(request, key):
     else:
         full_key = f"{site.key}/{key}"
 
-    logger.warn(site)
     return render_page(request, site, full_key)
 #+end_src
 
+This =render_page= function is shared between the =index= request and the more complicated route handler. 
+
+It's manually instrumented with a few [[https://prometheus.github.io/client_python/][Prometheus Client]] counters and gauges to be emitted on top of what comes out of =django-prometheus= already. This extra instrumentation is just enough to make a per-site and per-page hit chart, along with some very rudimentary [[id:20240213T120603.921365][User-Agent break-down]] to filter out most of the automated traffic.
+
 #+begin_src python :tangle arcology/views.py
+page_counter = Counter("arcology_page", "Hit counter for each page", ["site", "page", "status", "agent_type"])
+render_latency = Histogram("arcology_page_render_seconds", "Latency for render_page func.", ["page", "site", "agent_type"])
+
+from arcology.agent_utils import AgentClassification
+
 def render_page(request, site, full_key):
-    the_page = get_object_or_404(Page, route_key=full_key)
-    links = the_page.collect_links()
-    page_html = the_page.to_html(links)
+    agent = AgentClassification.from_request(request)
 
-    return render(request, "arcology/page.html", dict(
-        site=site,
-        page=the_page,
+    with render_latency.labels(page=full_key, site=site.key, agent_type=agent).time():
+        try:
+            the_page = Page.objects.get(route_key=full_key)
+        except Page.DoesNotExist:
+            page_counter.labels(page=full_key, status=404, site=site.key, agent_type=agent).inc() 
+            raise Http404
+        links = the_page.collect_links()
+        page_html = the_page.to_html(links)
 
-        head_title=f"{the_page.title} - {site.title}",
-        html_content=page_html,
+        page_counter.labels(page=full_key, status=200, site=site.key, agent_type=agent).inc()
 
-        backlinks=the_page.collect_backlinks(),
-        keywords=the_page.collect_keywords().all(),
-        references=the_page.collect_references(),
-    ))
+        return render(request, "arcology/page.html", dict(
+            site=site,
+            page=the_page,
+
+            head_title=f"{the_page.title} - {site.title}",
+            html_content=page_html,
+
+            backlinks=the_page.collect_backlinks(),
+            keywords=the_page.collect_keywords().all(),
+            references=the_page.collect_references(),
+        ))
 #+end_src
 
-*** Rendering the page
+*** Rendering the converted Org HTML in to a whole web-page
 
 The =page= template extends the app template defined below, which provides four blocks to inject content in to:
 
@@ -680,7 +697,7 @@ The main =content= block contains the =<main>= generated by the native parser, a
 {% endblock %}
 #+end_src
 
-*** Org Page Stylings
+*** Org Page-specific CSS Stylings
 
 Most of the page CSS is defined below, but the content CSS is here, nearer the actual implementation of the flexbox:
 

arcology/agent_utils.py

@@ -0,0 +1,107 @@
+# [[file:../scaffolding.org::*User-Agent break-down][User-Agent break-down:1]]
+from __future__ import annotations
+import logging
+from typing import List
+from enum import Enum
+
+logger = logging.getLogger(__name__)
+
+class AgentClassification(str, Enum):
+  NO_UA = "no-ua"
+  UNKNOWN = "unknown"
+  INTERNAL = "internal"
+  BROWSER = "browser"
+  MATRIX = "matrix"
+  APP = "app"
+  FEDIVERSE = "fediverse"
+  FEED = "feed"
+  BOT = "bot"
+  AUTOMATION = "automation"
+  
+
+  def __str__(self):
+    return self.value
+
+  @classmethod
+  def from_request(cls, request) -> AgentClassification:
+    user_agent = request.headers.get("User-Agent")
+    if user_agent == "":
+      return cls.NO_UA
+    if 'prometheus' in user_agent:
+        return cls.INTERNAL
+    if 'feediverse' in user_agent:
+        return cls.INTERNAL
+    if 'Chrome/' in user_agent:
+        return cls.BROWSER
+    if 'Firefox/' in user_agent:
+        return cls.BROWSER
+    if 'DuckDuckGo/' in user_agent:
+        return cls.BROWSER
+    if 'Safari/' in user_agent:
+        return cls.BROWSER
+    if 'Synapse' in user_agent:
+        return cls.MATRIX
+    if 'Element' in user_agent:
+        return cls.MATRIX
+    if 'SubwayTooter' in user_agent:
+        return cls.APP
+    if 'Dalvik' in user_agent:
+        return cls.APP
+    if 'Nextcloud-android' in user_agent:
+        return cls.APP
+    if 'Pleroma' in user_agent:
+        return cls.FEDIVERSE
+    if 'Mastodon/' in user_agent:
+        return cls.FEDIVERSE
+    if 'Akkoma' in user_agent:
+        return cls.FEDIVERSE
+    if 'Friendica' in user_agent:
+        return cls.FEDIVERSE
+    if 'FoundKey' in user_agent:
+        return cls.FEDIVERSE
+    if 'MissKey' in user_agent:
+        return cls.FEDIVERSE
+    if 'CalcKey' in user_agent:
+        return cls.FEDIVERSE
+    if 'gotosocial' in user_agent:
+        return cls.FEDIVERSE
+    if 'Epicyon' in user_agent:
+        return cls.FEDIVERSE
+    if 'feedparser' in user_agent:
+        return cls.FEED
+    if 'granary' in user_agent:
+        return cls.FEED
+    if 'Tiny Tiny RSS' in user_agent:
+        return cls.FEED
+    if 'Go_NEB' in user_agent:
+        return cls.FEED
+    if 'Gwene' in user_agent:
+        return cls.FEED
+    if 'Feedbin' in user_agent:
+        return cls.FEED
+    if 'SimplePie' in user_agent:
+        return cls.FEED
+    if 'Elfeed' in user_agent:
+        return cls.FEED
+    if 'inoreader' in user_agent:
+        return cls.FEED
+    if 'Reeder' in user_agent:
+        return cls.FEED
+    if 'Miniflux' in user_agent:
+        return cls.FEED
+    if 'Bot' in user_agent:
+        return cls.BOT
+    if 'bot' in user_agent:
+        return cls.BOT
+    if 'Poduptime' in user_agent:
+        return cls.BOT
+    if 'curl/' in user_agent:
+        return cls.AUTOMATION
+    if 'wget/' in user_agent:
+        return cls.AUTOMATION
+    
+
+    logger.warn(f"Unknown User-Agent: {user_agent}")
+
+    return cls.UNKNOWN
+# User-Agent break-down:1 ends here

arcology/templates/arcology/page.html

@@ -1,12 +1,12 @@
-{# [[file:../../../arcology.org::*Rendering the page][Rendering the page:1]] #}
+{# [[file:../../../arcology.org::*Rendering the converted Org HTML in to a whole web-page][Rendering the converted Org HTML in to a whole web-page:1]] #}
 {% extends "arcology/app.html" %}
-{# Rendering the page:1 ends here #}
+{# Rendering the converted Org HTML in to a whole web-page:1 ends here #}
 
-{# [[file:../../../arcology.org::*Rendering the page][Rendering the page:2]] #}
+{# [[file:../../../arcology.org::*Rendering the converted Org HTML in to a whole web-page][Rendering the converted Org HTML in to a whole web-page:2]] #}
 {% block title %}{{ head_title }}{% endblock %}
-{# Rendering the page:2 ends here #}
+{# Rendering the converted Org HTML in to a whole web-page:2 ends here #}
 
-{# [[file:../../../arcology.org::*Rendering the page][Rendering the page:3]] #}
+{# [[file:../../../arcology.org::*Rendering the converted Org HTML in to a whole web-page][Rendering the converted Org HTML in to a whole web-page:3]] #}
 {% block extra_head %}
   {% for feed in feeds %}
     <link rel="alternate" type="application/atom+xml" href="{{ feed.url }}" title="{{ feed.title }}" />
@@ -17,9 +17,9 @@
     <meta name="robots" content=""/>
   {% endif %}
 {% endblock %}
-{# Rendering the page:3 ends here #}
+{# Rendering the converted Org HTML in to a whole web-page:3 ends here #}
 
-{# [[file:../../../arcology.org::*Rendering the page][Rendering the page:4]] #}
+{# [[file:../../../arcology.org::*Rendering the converted Org HTML in to a whole web-page][Rendering the converted Org HTML in to a whole web-page:4]] #}
 {% block content %}
   {# HTML is sent through without HTML Escaping via | safe #}
   {{ html_content | safe }}
@@ -59,4 +59,4 @@
     {% endif %}
   </section>
 {% endblock %}
-{# Rendering the page:4 ends here #}
+{# Rendering the converted Org HTML in to a whole web-page:4 ends here #}

arcology/views.py

@@ -1,22 +1,22 @@
 # [[file:../arcology.org::*The Web Server][The Web Server:2]]
 import logging
-from django.http import HttpResponse, HttpResponseNotFound
+from django.http import HttpResponse, HttpResponseNotFound, Http404
 from django.shortcuts import render, get_object_or_404
 
 from arcology.models import Page, Feed, Site
 from roam.models import Link
 
+from prometheus_client import Counter, Histogram
+
 logger = logging.getLogger(__name__)
 # The Web Server:2 ends here
 
 # [[file:../arcology.org::*=GET /= site index][=GET /= site index:1]]
 def index(request):
     site = Site.from_request(request)
-    logger.warn(site)
 
     full_key = f"{site.key}/index"
 
-    logger.warn(full_key)
     return render_page(request, site, full_key)
 # =GET /= site index:1 ends here
 
@@ -30,27 +30,40 @@ def org_page(request, key):
     else:
         full_key = f"{site.key}/{key}"
 
-    logger.warn(site)
     return render_page(request, site, full_key)
 # Arcology Org Page handler:1 ends here
 
 # [[file:../arcology.org::*Arcology Org Page handler][Arcology Org Page handler:2]]
+page_counter = Counter("arcology_page", "Hit counter for each page", ["site", "page", "status", "agent_type"])
+render_latency = Histogram("arcology_page_render_seconds", "Latency for render_page func.", ["page", "site", "agent_type"])
+
+from arcology.agent_utils import AgentClassification
+
 def render_page(request, site, full_key):
-    the_page = get_object_or_404(Page, route_key=full_key)
-    links = the_page.collect_links()
-    page_html = the_page.to_html(links)
+    agent = AgentClassification.from_request(request)
 
-    return render(request, "arcology/page.html", dict(
-        site=site,
-        page=the_page,
+    with render_latency.labels(page=full_key, site=site.key, agent_type=agent).time():
+        try:
+            the_page = Page.objects.get(route_key=full_key)
+        except Page.DoesNotExist:
+            page_counter.labels(page=full_key, status=404, site=site.key, agent_type=agent).inc() 
+            raise Http404
+        links = the_page.collect_links()
+        page_html = the_page.to_html(links)
 
-        head_title=f"{the_page.title} - {site.title}",
-        html_content=page_html,
+        page_counter.labels(page=full_key, status=200, site=site.key, agent_type=agent).inc()
 
-        backlinks=the_page.collect_backlinks(),
-        keywords=the_page.collect_keywords().all(),
-        references=the_page.collect_references(),
-    ))
+        return render(request, "arcology/page.html", dict(
+            site=site,
+            page=the_page,
+
+            head_title=f"{the_page.title} - {site.title}",
+            html_content=page_html,
+
+            backlinks=the_page.collect_backlinks(),
+            keywords=the_page.collect_keywords().all(),
+            references=the_page.collect_references(),
+        ))
 # Arcology Org Page handler:2 ends here
 
 # [[file:../arcology.org::*Atom Feed Handler][Atom Feed Handler:1]]

default.nix

@@ -18,14 +18,17 @@ python3.pkgs.buildPythonPackage rec {
   propagatedBuildInputs = (with pkgs; [
     arroyo_rs
   ]) ++ (with python3.pkgs; [
-    click
-    setuptools
-    django
-    polling
-    django-prometheus
     arrow
+    click
+    django
+    django-prometheus
+    gunicorn
+    polling
+    setuptools
   ]);
 
+  passthru.gunicorn = python3.pkgs.gunicorn;
+
   meta = with lib; {
     description = "An org-mode site engine";
     homepage = "https://engine.arcology.garden/";

deployment.org

@@ -0,0 +1,226 @@
+:PROPERTIES:
+:ID:       20240213T124300.774781
+:END:
+#+TITLE: Deploying the Arcology
+#+filetags: :Project:
+
+#+ARCOLOGY_KEY: arcology/django/deploy
+#+ARROYO_NIXOS_MODULE: nixos/arcology2.nix
+#+ARROYO_NIXOS_MODULE: nixos/arcology2-module.nix
+#+ARROYO_NIXOS_ROLE: server
+
+* NEXT Bootstrapping on non-NixOS
+
+- =nix run= ingestfiles and generator commands to stand up a system. core for [[id:20231113T195508.942155][Rebuild of The Complete Computer]].
+
+* Running on [[id:20211120T220054.226284][The Wobserver]], self-hosting Arcology with the [[id:arroyo/django/generators][The Arroyo Generators]]
+
+Package building is handled in the [[id:arcology/django/scaffolding][Arcology Project Scaffolding]]..
+
+** NixOS module
+
+provide it in the =flake.nix= too...
+
+We need two services, the =watchsync= server and a =gunicorn=. We need a virtualhost.
+
+#+begin_src nix :tangle ~/arroyo-nix/nixos/arcology2-module.nix
+{ lib, config, pkgs, ... }:
+
+with pkgs;
+with lib;
+let
+  cfg = config.services.arcology-ng;
+
+  # might want to generate a localsettings.py or so to configure the application for deployment, rather than use process env
+
+  env = {
+    ARCOLOGY_ENV = cfg.environment;
+    ARCOLOGY_DIRECTORY = cfg.orgDir;
+
+    ARCOLOGY_DB = "${cfg.dataDir}/databases/arcology2.db";
+    # ARCOLOGY_SYNCTHING_KEY=
+    # ALLOWED_HOSTS in to settings based on cfg.domains...
+
+    GUNICORN_CMD_ARGS = "--bind=${cfg.address}:${toString cfg.port} -w ${toString cfg.workerCount}";
+  };
+
+  svcConfig = {
+    systemd.services.arcology2-watchsync = {
+      description = "Arcology Django Syncthing Watcher";
+      after = ["network.target"];
+      wantedBy = ["multi-user.target"];
+      environment = env;
+      preStart = ''
+        ${cfg.packages.arcology}/bin/arcology migrate
+        ${cfg.packages.arcology}/bin/arcology seed
+      '';
+      script = ''
+        ${cfg.packages.arcology}/bin/arcology watchsync
+      '';
+      serviceConfig = {
+        Type = "simple";
+        User = "arcology";
+        Group = "arcology";
+        WorkingDirectory = cfg.dataDir;
+        # hardening...
+      };
+    };
+    systemd.services.arcology2-web = {
+      description = "Arcology Django Gunicorn";
+      after = ["network.target"];
+      wantedBy = ["multi-user.target"];
+      environment = env;
+      script = ''
+        ${cfg.packages.arcology.gunicorn}/bin/gunicorn arcology.wsgi
+      '';
+      serviceConfig = {
+        Type = "simple";
+        User = "arcology";
+        Group = "arcology";
+        WorkingDirectory = cfg.dataDir;
+        # hardening...
+      };
+    };
+  };
+
+  domainVHosts = {
+    services.nginx.virtualHosts."${head cfg.domains}" = mkIf (cfg.enable && cfg.generateVirtualHosts) {
+      serverAliases = tail cfg.domains;
+      locations."/".proxyPass = "http://${cfg.address}:${toString cfg.port}";
+      locations."/".extraConfig = ''
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Forwarded-Host $http_host;
+        proxy_set_header Host $host;
+      '';
+    };
+  };
+
+  userConfig = (mkIf (!config.services.arcology.enable) {
+    ids.uids.arcology = 900;
+    ids.gids.arcology = 900;
+
+    users.users.arcology = {
+      group = "arcology";
+      home = cfg.dataDir;
+      createHome = true;
+      shell = "${bash}/bin/bash";
+      isSystemUser = true;
+      uid = config.ids.uids.arcology;
+    };
+
+    users.groups.arcology = {
+      gid = config.ids.gids.arcology;
+    };
+  });
+in {
+  options = {
+    services.arcology-ng = {
+      enable = mkEnableOption "arcology-ng";
+
+      packages.arcology = mkOption {
+        type = types.package;
+        description = mdDoc ''
+        '';
+      };
+
+      domains = mkOption {
+        type = types.listOf types.str;
+      };
+
+      address = mkOption {
+        type = types.str;
+        default = "localhost";
+        description = lib.mdDoc "Web interface address.";
+      };
+
+      port = mkOption {
+        type = types.port;
+        default = 29543;
+        description = lib.mdDoc "Web interface port.";
+      };
+
+      environment = mkOption {
+        type = types.enum ["prod" "dev"];
+        default = "prod";
+      };
+
+      workerCount = mkOption {
+        type = types.number;
+        default = 16;
+        description = lib.mdDoc "gunicorn worker count; they recommend 2-4 workers per core.";
+      };
+
+      generateVirtualHosts = mkOption {
+        type = types.bool;
+        default = true;
+        description = lib.mdDoc "control whether nginx virtual hosts should be created";
+      };
+
+      dataDir = mkOption {
+        type = types.path;
+        default = "/var/lib/arcology";
+        description = mdDoc ''
+          Directory to store Arcology cache files, database, etc. Service User's home directory.
+        '';
+      };
+
+      orgDir = mkOption {
+        type = types.path;
+        description = mdDoc ''
+          Directory containing the org-mode documents.
+          Arcology needs read-only access to this directory.
+        '';
+      };
+    };
+  };
+
+  config = mkIf cfg.enable (mkMerge [
+    svcConfig 
+    domainVHosts 
+    # prevent double-definition of user entities from previous service's manifest
+    # yanked directly from arcology-fastapi
+    userConfig
+  ]);
+}
+#+end_src
+
+*** INPROGRESS finish this
+:LOGBOOK:
+- State "INPROGRESS" from "NEXT"       
+:END:
+*** NEXT validate the environment variables are used
+*** NEXT consider generating a local settings.py with our configuration overrides
+*** NEXT figure out better way to call in to Arcology and Arroyo in the service definition
+
+probably just =getFlake= but augh.
+
+*** NEXT service hardening
+*** NEXT static files under gunicorn/nginx
+*** NEXT secret infrastructure for the syncthing key
+
+or a way to load that in to the DB 🤔
+
+** Arroyo NixOS snippet & my deployment configuration
+
+The [[id:arroyo/nixos][Arroyo NixOS Generator]] will integrate this or you can do this yourself. just import the nixos module above, and then configure it as below:
+
+#+begin_src nix :tangle ~/arroyo-nix/nixos/arcology2.nix
+{ pkgs, ... }:
+
+let
+  arroyo_rs = pkgs.callPackage /home/rrix/org/arroyo/default.nix {};
+  arcology = pkgs.callPackage /home/rrix/org/arcology-django/default.nix { inherit arroyo_rs; };
+in {
+  services.arcology-ng = {
+    enable = true;
+    packages.arcology = arcology;
+    domains = ["engine2.arcology.garden" "v2.thelionsrear.com" "v2.arcology.garden" "cce2.whatthefuck.computer"];
+    orgDir = "/media/org";
+    dataDir = "/srv/arcology";
+  };
+}
+#+end_src
+
+*** NEXT the package import needs to be much better than this.

pyproject.toml

@@ -5,7 +5,7 @@ version = "0.0.1"
 description = "org-mode metadata query engine, publishing platform, and computer metaprogrammer"
 # license = "Hey Smell This"
 readme = "README.md"
-dependencies = ["click ~=8.1", "django ~= 4.2", "django-stub", "polling", "django-prometheus", "arroyo", "arrow ~= 1.3.0"]
+dependencies = ["click ~=8.1", "django ~= 4.2", "django-stub", "polling", "django-prometheus", "arroyo", "arrow ~= 1.3.0", "gunicorn ~= 21.0"]
 requires-python = ">=3.10"
 authors = [
     { name = "Ryan Rix", email = "code@whatthefuck.computer" }
@@ -17,6 +17,9 @@ authors = [
 [tool.setuptools]
 package-dir = {"" = "."}
 
+[tool.setuptools.package-data]
+arcology = ['settings/sites.json']
+
 [tool.setuptools.packages.find]
 where = ["."]
 

scaffolding.org

@@ -17,7 +17,7 @@ version = "0.0.1"
 description = "org-mode metadata query engine, publishing platform, and computer metaprogrammer"
 # license = "Hey Smell This"
 readme = "README.md"
-dependencies = ["click ~=8.1", "django ~= 4.2", "django-stub", "polling", "django-prometheus", "arroyo", "arrow ~= 1.3.0"]
+dependencies = ["click ~=8.1", "django ~= 4.2", "django-stub", "polling", "django-prometheus", "arroyo", "arrow ~= 1.3.0", "gunicorn ~= 21.0"]
 requires-python = ">=3.10"
 authors = [
     { name = "Ryan Rix", email = "code@whatthefuck.computer" }
@@ -29,6 +29,9 @@ authors = [
 [tool.setuptools]
 package-dir = {"" = "."}
 
+[tool.setuptools.package-data]
+arcology = ['settings/sites.json']
+
 [tool.setuptools.packages.find]
 where = ["."]
 
@@ -41,7 +44,7 @@ build-backend = "setuptools.build_meta"
 
 =nix build= will spit out a python project that can be used in a NixOS definition. now where would we get one of those...? It's marked with =licenses.unfree= right now because I don't think [[id:20220116T143655.499306][Hey Smell This]] will pass the OSI sniff-test.
 
-#+begin_src python :tangle default.nix
+#+begin_src nix :tangle default.nix
 {
   pkgs ? import <nixpkgs> {},
   lib  ? pkgs.lib,
@@ -61,14 +64,17 @@ python3.pkgs.buildPythonPackage rec {
   propagatedBuildInputs = (with pkgs; [
     arroyo_rs
   ]) ++ (with python3.pkgs; [
-    click
-    setuptools
-    django
-    polling
-    django-prometheus
     arrow
+    click
+    django
+    django-prometheus
+    gunicorn
+    polling
+    setuptools
   ]);
 
+  passthru.gunicorn = python3.pkgs.gunicorn;
+
   meta = with lib; {
     description = "An org-mode site engine";
     homepage = "https://engine.arcology.garden/";
@@ -82,7 +88,7 @@ python3.pkgs.buildPythonPackage rec {
 
 Nix is really going this direction, I'm not sure it's worthwhile but I'm going to see how to adapt to this world. It should be possible to =nix run= a few apps to be able to operate the arcology.
 
-#+begin_src python :tangle flake.nix
+#+begin_src nix :tangle flake.nix
 {
   description = "Arcology Site Engine, Django Edition";
 
@@ -131,7 +137,7 @@ Nix is really going this direction, I'm not sure it's worthwhile but I'm going t
 
 =nix develop= or =nix-shell= will set you up with an environment that has Python programming dependencies available.
 
-#+begin_src python :tangle shell.nix
+#+begin_src nix :tangle shell.nix
 { pkgs ? import <nixpkgs> {},
   python3 ? pkgs.python3,
 
@@ -143,13 +149,14 @@ let
     pytest
     mypy
 
+    arrow
     arroyo_rs
     django_4
-    django-stubs.override { django = django_4; }
-    django-stubs-ext.override { django = django_4; }
-    polling
     django-prometheus
-    arrow
+    django-stubs-ext.override { django = django_4; }
+    django-stubs.override { django = django_4; }
+    gunicorn
+    polling
   ]);
 in pkgs.mkShell {
   packages = (with pkgs; [
@@ -191,10 +198,6 @@ static/arcology/fonts/Vulf*
 
 * Django bootstraps
 
-tbh don't worry about these.
-
-shouts out to [[https://lincolnloop.com/insights/using-pyprojecttoml-in-your-django-project/][peter baumgautner at lincoln loop or whoever wrote this]]:
-
 #+begin_src python :tangle arcology/__init__.py
 import os
 import sys
@@ -207,7 +210,7 @@ def django_manage():
 
 this and a bit in =pyproject.toml= lets you just type =arcology watchfiles= to invoke a manage.py command.
 
-These are generated scaffolds for now.
+These are generated scaffolds for now, basically the manage.py and -m arcology are the same and that is annoying, but i'll fix it some day.
 
 #+begin_src python :tangle manage.py
 #!/nix/store/c3cjxhn73xa5s8fm79w95d0879bijp04-python3-3.10.13/bin/python
@@ -271,3 +274,100 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "arcology.settings")
 
 application = get_wsgi_application()
 #+end_src
+
+* Middlewares
+** User-Agent break-down
+:PROPERTIES:
+:ID:       20240213T120603.921365
+:END:
+
+This =AgentClassification= enumeration class can take a User Agent header and map it to one of a handful of groups, which a user has the ability to extend. =AgentClassification.from_request(request)= will return a string from an enumeration, this is probably useful in labeling metrics or site statistics.
+
+
+#+NAME: agent_classifications
+| User Agent Substring | Enumeration |
+|----------------------+-------------|
+| prometheus           | INTERNAL    |
+| feediverse           | INTERNAL    |
+| Chrome/              | BROWSER     |
+| Firefox/             | BROWSER     |
+| DuckDuckGo/          | BROWSER     |
+| Safari/              | BROWSER     |
+| Synapse              | MATRIX      |
+| Element              | MATRIX      |
+| SubwayTooter         | APP         |
+| Dalvik               | APP         |
+| Nextcloud-android    | APP         |
+| Pleroma              | FEDIVERSE   |
+| Mastodon/            | FEDIVERSE   |
+| Akkoma               | FEDIVERSE   |
+| Friendica            | FEDIVERSE   |
+| FoundKey             | FEDIVERSE   |
+| MissKey              | FEDIVERSE   |
+| CalcKey              | FEDIVERSE   |
+| gotosocial           | FEDIVERSE   |
+| Epicyon              | FEDIVERSE   |
+| feedparser           | FEED        |
+| granary              | FEED        |
+| Tiny Tiny RSS        | FEED        |
+| Go_NEB               | FEED        |
+| Gwene                | FEED        |
+| Feedbin              | FEED        |
+| SimplePie            | FEED        |
+| Elfeed               | FEED        |
+| inoreader            | FEED        |
+| Reeder               | FEED        |
+| Miniflux             | FEED        |
+| Bot                  | BOT         |
+| bot                  | BOT         |
+| Poduptime            | BOT         |
+| curl/                | AUTOMATION  |
+| wget/                | AUTOMATION  |
+
+#+begin_src python :tangle arcology/agent_utils.py :noweb yes
+from __future__ import annotations
+import logging
+from typing import List
+from enum import Enum
+
+logger = logging.getLogger(__name__)
+
+class AgentClassification(str, Enum):
+  NO_UA = "no-ua"
+  UNKNOWN = "unknown"
+  <<make_enum()>>
+
+  def __str__(self):
+    return self.value
+
+  @classmethod
+  def from_request(cls, request) -> AgentClassification:
+    user_agent = request.headers.get("User-Agent")
+    if user_agent == "":
+      return cls.NO_UA
+    <<agent_classifier()>>
+
+    logger.warn(f"Unknown User-Agent: {user_agent}")
+
+    return cls.UNKNOWN
+#+end_src
+
+#+name: make_enum
+#+begin_src emacs-lisp :var tbl=agent_classifications
+(thread-last
+  tbl
+  (mapcar (pcase-lambda (`(,substring ,enum)) enum))
+  (-uniq)
+  (mapcar (lambda (enum) (format "%s = \"%s\"\n" enum (downcase enum))))
+  (apply #'concat))
+#+end_src
+
+#+name: agent_classifier
+#+begin_src emacs-lisp :var tbl=agent_classifications
+(thread-last
+  tbl
+  (mapcar (pcase-lambda (`(,substring ,enum))
+            (concat "if '" substring "' in user_agent:" "\n"
+                    "    return cls." enum "\n")))
+  (apply #'concat))
+#+end_src

shell.nix

@@ -10,13 +10,14 @@ let
     pytest
     mypy
 
+    arrow
     arroyo_rs
     django_4
-    django-stubs.override { django = django_4; }
-    django-stubs-ext.override { django = django_4; }
-    polling
     django-prometheus
-    arrow
+    django-stubs-ext.override { django = django_4; }
+    django-stubs.override { django = django_4; }
+    gunicorn
+    polling
   ]);
 in pkgs.mkShell {
   packages = (with pkgs; [

static/arcology/css/app.css

@@ -1,4 +1,4 @@
-/* [[file:../../../arcology.org::*Org Page Stylings][Org Page Stylings:1]] */
+/* [[file:../../../arcology.org::*Org Page-specific CSS Stylings][Org Page-specific CSS Stylings:1]] */
 .content {
   margin-left: auto;
   margin-right: auto;
@@ -23,9 +23,9 @@
   flex-shrink: 1;
   flex-basis: 30ch;
 }
-/* Org Page Stylings:1 ends here */
+/* Org Page-specific CSS Stylings:1 ends here */
 
-/* [[file:../../../arcology.org::*Org Page Stylings][Org Page Stylings:2]] */
+/* [[file:../../../arcology.org::*Org Page-specific CSS Stylings][Org Page-specific CSS Stylings:2]] */
 section.sidebar {
   display: flex;
   flex-flow: column wrap;
@@ -34,9 +34,9 @@ section.sidebar {
 section.sidebar > div.backlinks {
   flex-grow: 1;
 }
-/* Org Page Stylings:2 ends here */
+/* Org Page-specific CSS Stylings:2 ends here */
 
-/* [[file:../../../arcology.org::*Org Page Stylings][Org Page Stylings:3]] */
+/* [[file:../../../arcology.org::*Org Page-specific CSS Stylings][Org Page-specific CSS Stylings:3]] */
 .content::before {
     align-self: stretch;
     content: '';
@@ -47,7 +47,7 @@ section.sidebar > div.backlinks {
 .content > *:first-child {
     order: -1;
 }
-/* Org Page Stylings:3 ends here */
+/* Org Page-specific CSS Stylings:3 ends here */
 
 /* [[file:../../../arcology.org::*CSS][CSS:1]] */
 body {