arcology/generators.org

:PROPERTIES:
:ID:       arroyo/django/generators
:END:
#+TITLE: The Arroyo Generators
#+filetags: :Project:
#+ARCOLOGY_KEY: arcology/code-generators

#+begin_src python :tangle generators/__init__.py
#+end_src

talk about what these modules are used to do, show the management commands from the usage section again.

* Arroyo Generator Data Models

** NEXT document the Generator pattern described by the Abstract Base Class.

#+begin_src python :tangle generators/models.py
from __future__ import annotations
from typing import List
from django.db import models
from django.conf import settings
from django_prometheus.models import ExportModelOperationsMixin as EMOM

import arroyo.arroyo_rs as native

import roam.models

import pathlib
import graphlib

import logging
logger = logging.getLogger(__name__)


class GeneratorRole(EMOM('generatorRole'), models.Model):
    name = models.CharField(max_length=512, primary_key=True)

    @classmethod
    def get_or_create_many(cls, names: List[str]) -> List[GeneratorRole]:
        return [
            cls.objects.get_or_create(
                name=name
            )[0]
            for name in names
        ]


class Generator(EMOM('generator'), models.Model):
    class Meta:
        abstract = True

    ARROYO_KEYWORD_NAME = "THE_ABC"

    org_file = models.ForeignKey(
        roam.models.File,
        on_delete=models.CASCADE,
    )
    destination = models.CharField(max_length=512)

    roles = models.ManyToManyField(
        GeneratorRole,
        related_name="THE_ABC",
    )
    excluded_roles = models.ManyToManyField(
        GeneratorRole,
        related_name="THE_ABC",
    )

    def to_babel(self):
        raise NotImplemented

    def _to_babel_list(self, **kwargs):
        formatter = kwargs.get('formatter', "{}")
        return formatter.format(self.destination)

    def _to_babel_inclusion(self, **kwargs):
        formatter = kwargs.get('formatter', "{}")
        path = formatter.format(self.destination)
        with open(path, 'r') as f:
            return f.read()

    def __str__(self):
        return self.__repr__()

    def __repr__(self):
        return f"<{self.__class__.__name__}: {self.org_file.path} -> {self.destination}>"

    @classmethod
    def collect_for_babel(cls, **kwargs):
        return cls.objects.filter(roles__in=[kwargs['role']])

    @classmethod
    def create_from_arroyo(cls, doc: native.Document) -> List[Generator]:
        role_names = doc.collect_keywords("ARROYO_SYSTEM_ROLE") + doc.collect_keywords("ARROYO_ROLE")
        excluded_role_names = doc.collect_keywords("ARROYO_SYSTEM_EXCLUDE") + doc.collect_keywords("ARROYO_EXCLUDE_ROLE")
        f = roam.models.File.objects.get(path=doc.path)
        destinations = doc.collect_keywords(cls.ARROYO_KEYWORD_NAME)

        if len(destinations) == 0:
            # logger.debug(f"{cls} Skip {doc.path}")
            return []

        logger.debug(f"{cls} {doc.path} -> {destinations}")

        excluded_roles = GeneratorRole.get_or_create_many(excluded_role_names)
        if role_names == []:
            roles = [role for role in GeneratorRole.objects.all() if role not in excluded_roles]
        else:
            roles = GeneratorRole.get_or_create_many(role_names)

        ret = []
        for destination in destinations:
            obj = cls.objects.get_or_create(
                org_file=f,
                destination=destination,
            )[0]
            obj.roles.set(roles)
            obj.excluded_roles.set(excluded_roles)
            ret.append(obj)
        return ret
#+end_src

*** NEXT Testing

Test the classmethods =create_from_arroyo= and =collect_for_babel= and the =to_babel= instance method

role list and exclusions in =create_from_arroyo=

** Generate an Emacs configuration

Emacs configuration is not declarative but imperative so dependency order needs to be established which the Nix modules do not nee to perform. =#+ARROYO_MODULE_WANTS= and =#+ARROYO_MODULE_WANTED= Keywords allow documents to describe dependency relationships. The python built-in [[https://docs.python.org/3/library/graphlib.html][=graphlib=]] is used to order the =EmacsSnippets=, with the =DependencyRelationship= class caching edges between the snippets in the database.

#+begin_src python :tangle generators/models.py
class DependencyRelationship(EMOM('dependencyRelationship'), models.Model):
    # extract enough information to run https://docs.python.org/3/library/graphlib.html on the emacs snippets...
    dependency = models.ForeignKey(
        "EmacsSnippet",
        on_delete=models.CASCADE,
        blank=True,
        related_name="dependent_relations",
    )
    dependent = models.ForeignKey(
        "EmacsSnippet",
        on_delete=models.CASCADE,
        blank=True,
        related_name="dependancy_relations",
    )

    dependency_path = models.CharField(max_length=512)
    dependent_path = models.CharField(max_length=512)

    @classmethod
    def extract_ordering_from_arroyo(cls, ts: graphlib.TopologicalSorter, doc: native.Document):
        base_path_abs = pathlib.Path(settings.ARCOLOGY_BASE_DIR).expanduser()
        dependencies = doc.collect_keywords("ARROYO_MODULE_WANTS")
        dependents = doc.collect_keywords("ARROYO_MODULE_WANTED")

        ts.add(doc.path)

        for dep in dependencies:
            dep_path = str(base_path_abs.joinpath(dep))
            logger.debug(f"EXTRACT {doc.path} wants {dep_path}")
            ts.add(doc.path, dep_path)

        for dep in dependents:
            dep_path = str(base_path_abs.joinpath(dep))
            logger.debug(f"EXTRACT(rdep) {dep_path} wants {doc.path}")
            ts.add(doc.path, dep_path)

    @classmethod
    def collect_for_babel(cls) -> List[EmacsSnippet]:
        ts = graphlib.TopologicalSorter()
        mapping = dict()
        # add all the nodes from EmacsSnippet
        for snippet in EmacsSnippet.objects.all():
            path = snippet.org_file.path
            ts.add(path)
            mapping[path] = snippet
        # add all the DependencyRelationships
        for rel in cls.objects.all():
            pred = rel.dependency
            succ = rel.dependent
            ts.add(succ.org_file.path, pred.org_file.path)

        return list(map(lambda it: mapping[it], ts.static_order()))


class EmacsSnippet(EMOM('emacs_snippet'), Generator):
    ARROYO_KEYWORD_NAME = "ARROYO_EMACS_MODULE"

    roles = models.ManyToManyField(
        GeneratorRole,
        related_name="emacs_modules",
    )
    excluded_roles = models.ManyToManyField(
        GeneratorRole,
        related_name="excluded_emacs_modules",
    )

    dependencies = models.ManyToManyField(
        "EmacsSnippet",
        related_name="dependents",
        through="DependencyRelationship",
        through_fields=("dependency", "dependent"),
    )

    def to_babel(self) -> str:
        formatter = f"{settings.ARCOLOGY_EMACS_SNIPPETS_DIR}/{{}}.el"
        return self._to_babel_inclusion(formatter=formatter)

    @classmethod
    def collect_for_babel(cls, **kwargs):
        return DependencyRelationship.collect_for_babel()

    @classmethod
    def create_from_arroyo(cls, doc: native.Document) -> List[Generator]:
        """
        create_from_arroyo is overridden to persist ordering rules
        """
        new_snippets = super(EmacsSnippet, cls).create_from_arroyo(doc)
        base_path_abs = pathlib.Path(settings.ARCOLOGY_BASE_DIR).expanduser()

        # convert these to relation model objects..., but which?
        # back to the problem of defining relations to objects which may not yet exist...
        deps = doc.collect_keywords("ARROYO_MODULE_WANTS")
        reverse_deps = doc.collect_keywords("ARROYO_MODULE_WANTED")

        for obj in new_snippets:
            for dep in deps:
                dep_path = base_path_abs.joinpath(dep)
                logger.debug(f"create dep from {doc.path} to {dep_path}")
                the_dep = EmacsSnippet.objects.get(org_file__path=dep_path)

                DependencyRelationship.objects.create(
                    dependency=the_dep,
                    dependent=obj,
                )
            for dep in reverse_deps:
                dep_path = base_path_abs.joinpath(dep)
                logger.debug(f"create rdep to {dep_path} from {doc.path}")
                the_dep = EmacsSnippet.objects.get(org_file__path=dep_path)
                DependencyRelationship.objects.create(
                    dependency=obj,
                    dependent=the_dep,
                )

        return new_snippets
#+end_src

*** NEXT Testing

Test the classmethods and =to_babel= instance method

test the depdendencyrelationship stuff, the graphlib sort.

** Emacs custom package overrides

It's possible for pages to stash Nix code snippets in to tangled files to make custom Emacs packages available to Arroyo Emacs using helpers like nixpkgs's =melpaBuild= and =trivialBuild= commands.

#+begin_src python :tangle generators/models.py
class EmacsEpkg(EMOM('emacs_epkg'), Generator):
    ARROYO_KEYWORD_NAME = "ARROYO_HOME_EPKGS"

    roles = models.ManyToManyField(
        GeneratorRole,
        related_name="emacs_packages",
    )
    excluded_roles = models.ManyToManyField(
        GeneratorRole,
        related_name="excluded_emacs_packages",
    )

    def to_babel(self) -> str:
        formatter = f"{settings.ARROYO_BASE_DIR}/{{}}"
        return self._to_babel_inclusion(formatter=formatter)
#+end_src

*** NEXT Testing

this one is pretty straightforward implementation of the ABC.

** Declarative NixOS module imports

It's possible for pages to signal all or certain deployment roles to include a new NixOS module. This allows a user to download an org-mode file in to their org-roam directory and it will and new functionality to their computer.

#+begin_src python :tangle generators/models.py
class NixosModule(EMOM('nixos_module'), Generator):
    ARROYO_KEYWORD_NAME = "ARROYO_NIXOS_MODULE"

    roles = models.ManyToManyField(
        GeneratorRole,
        related_name="nixos_modules",
    )
    excluded_roles = models.ManyToManyField(
        GeneratorRole,
        related_name="excluded_nixos_modules",
    )

    def to_babel(self) -> str:
        # XXX this is going to need to be configurable at some point.....
        formatter = "../../{}"
        return self._to_babel_list(formatter=formatter)
#+end_src

*** NEXT Testing

this one is pretty straightforward implementation of the ABC.

** Declarative =home-manager= module imports

It's possible to add home-manager modules to the user's profile in much the same way.

#+begin_src python :tangle generators/models.py
class HomeManagerModule(EMOM('home_manager_module'), Generator):
    ARROYO_KEYWORD_NAME = "ARROYO_HOME_MODULE"

    roles = models.ManyToManyField(
        GeneratorRole,
        related_name="home_modules",
    )
    excluded_roles = models.ManyToManyField(
        GeneratorRole,
        related_name="excluded_home_modules",
    )

    def to_babel(self) -> str:
        formatter = "{}"
        return self._to_babel_list(formatter=formatter)
#+end_src

*** NEXT Testing

this one is pretty straightforward implementation of the ABC.

** NEXT need to squash these

#+begin_src python :tangle generators/migrations/__init__.py
#+end_src

#+begin_src python :tangle generators/migrations/0001_initial.py
# Generated by Django 4.2.6 on 2023-11-27 05:50

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):
    initial = True

    dependencies = [
        ("roam", "0005_alter_link_dest_heading"),
    ]

    operations = [
        migrations.CreateModel(
            name="GeneratorRole",
            fields=[
                (
                    "name",
                    models.CharField(max_length=512, primary_key=True, serialize=False),
                ),
            ],
        ),
        migrations.CreateModel(
            name="NixosModule",
            fields=[
                (
                    "id",
                    models.BigAutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("destination", models.CharField(max_length=512)),
                (
                    "excluded_roles",
                    models.ManyToManyField(
                        related_name="excluded_nixos_modules",
                        to="generators.generatorrole",
                    ),
                ),
                (
                    "org_file",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE, to="roam.file"
                    ),
                ),
                (
                    "roles",
                    models.ManyToManyField(
                        related_name="nixos_modules", to="generators.generatorrole"
                    ),
                ),
            ],
            options={
                "abstract": False,
            },
        ),
        migrations.CreateModel(
            name="HomeManagerModule",
            fields=[
                (
                    "id",
                    models.BigAutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("destination", models.CharField(max_length=512)),
                (
                    "excluded_roles",
                    models.ManyToManyField(
                        related_name="excluded_home_modules",
                        to="generators.generatorrole",
                    ),
                ),
                (
                    "org_file",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE, to="roam.file"
                    ),
                ),
                (
                    "roles",
                    models.ManyToManyField(
                        related_name="home_modules", to="generators.generatorrole"
                    ),
                ),
            ],
            options={
                "abstract": False,
            },
        ),
        migrations.CreateModel(
            name="EmacsSnippet",
            fields=[
                (
                    "id",
                    models.BigAutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("destination", models.CharField(max_length=512)),
                (
                    "excluded_roles",
                    models.ManyToManyField(
                        related_name="excluded_emacs_modules",
                        to="generators.generatorrole",
                    ),
                ),
                (
                    "org_file",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE, to="roam.file"
                    ),
                ),
                (
                    "roles",
                    models.ManyToManyField(
                        related_name="emacs_modules", to="generators.generatorrole"
                    ),
                ),
            ],
            options={
                "abstract": False,
            },
        ),
        migrations.CreateModel(
            name="EmacsEpkg",
            fields=[
                (
                    "id",
                    models.BigAutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("destination", models.CharField(max_length=512)),
                (
                    "excluded_roles",
                    models.ManyToManyField(
                        related_name="excluded_emacs_packages",
                        to="generators.generatorrole",
                    ),
                ),
                (
                    "org_file",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE, to="roam.file"
                    ),
                ),
                (
                    "roles",
                    models.ManyToManyField(
                        related_name="emacs_packages", to="generators.generatorrole"
                    ),
                ),
            ],
            options={
                "abstract": False,
            },
        ),
    ]
#+end_src

#+begin_src python :tangle generators/migrations/0002_dependencyrelationship_emacssnippet_dependencies.py
# Generated by Django 4.2.6 on 2023-12-11 20:11

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):
    dependencies = [
        ("generators", "0001_initial"),
    ]

    operations = [
        migrations.CreateModel(
            name="DependencyRelationship",
            fields=[
                (
                    "id",
                    models.BigAutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("dependency_path", models.CharField(max_length=512)),
                ("dependant_path", models.CharField(max_length=512)),
                (
                    "dependant",
                    models.ForeignKey(
                        blank=True,
                        on_delete=django.db.models.deletion.CASCADE,
                        related_name="dependancy_relations",
                        to="generators.emacssnippet",
                    ),
                ),
                (
                    "dependency",
                    models.ForeignKey(
                        blank=True,
                        on_delete=django.db.models.deletion.CASCADE,
                        related_name="dependant_relations",
                        to="generators.emacssnippet",
                    ),
                ),
            ],
        ),
        migrations.AddField(
            model_name="emacssnippet",
            name="dependencies",
            field=models.ManyToManyField(
                related_name="dependants",
                through="generators.DependencyRelationship",
                to="generators.emacssnippet",
            ),
        ),
    ]
#+end_src

#+begin_src python :tangle generators/migrations/0003_rename_dependant_dependencyrelationship_dependent_and_more.py
# Generated by Django 4.2.6 on 2023-12-12 04:34

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):
    dependencies = [
        ("generators", "0002_dependencyrelationship_emacssnippet_dependencies"),
    ]

    operations = [
        migrations.RenameField(
            model_name="dependencyrelationship",
            old_name="dependant",
            new_name="dependent",
        ),
        migrations.RenameField(
            model_name="dependencyrelationship",
            old_name="dependant_path",
            new_name="dependent_path",
        ),
        migrations.AlterField(
            model_name="dependencyrelationship",
            name="dependency",
            field=models.ForeignKey(
                blank=True,
                on_delete=django.db.models.deletion.CASCADE,
                related_name="dependent_relations",
                to="generators.emacssnippet",
            ),
        ),
        migrations.AlterField(
            model_name="emacssnippet",
            name="dependencies",
            field=models.ManyToManyField(
                related_name="dependents",
                through="generators.DependencyRelationship",
                to="generators.emacssnippet",
            ),
        ),
    ]
#+end_src

* NEXT Module tests

#+begin_src python :tangle generators/tests.py
from django.test import TestCase

# Create your tests here.
#+end_src

* Admin

#+begin_src python :tangle generators/admin.py
from django.contrib import admin
from django import forms
from django.contrib.admin.widgets import FilteredSelectMultiple
from django.utils.translation import gettext as _

import generators.models


class DependencyInline(admin.TabularInline):
    model = generators.models.EmacsSnippet.dependencies.through
    fk_name = "dependent"

@admin.register(generators.models.EmacsEpkg)
@admin.register(generators.models.NixosModule)
@admin.register(generators.models.HomeManagerModule)
class GeneratorAdmin(admin.ModelAdmin):
    list_display = ["destination", "org_file"]
    filter_horizontal = ["excluded_roles", "roles"]

@admin.register(generators.models.EmacsSnippet)
class EmacsSnippetAdmin(admin.ModelAdmin):
    list_display = ["destination", "org_file"]
    filter_horizontal = ["excluded_roles", "roles"]
    inlines = [ DependencyInline ]

class RoleAdminForm(forms.ModelForm):
    emacs_modules = forms.ModelMultipleChoiceField(
        queryset=generators.models.EmacsSnippet.objects.all(),
        required=False,
        widget=FilteredSelectMultiple(
            verbose_name=_('Emacs Modules'),
            is_stacked=False
        )
    )
    emacs_packages = forms.ModelMultipleChoiceField(
        queryset=generators.models.EmacsEpkg.objects.all(),
        required=False,
        widget=FilteredSelectMultiple(
            verbose_name=_('Emacs Packages'),
            is_stacked=False
        )
    )
    nixos_modules = forms.ModelMultipleChoiceField(
        queryset=generators.models.NixosModule.objects.all(),
        required=False,
        widget=FilteredSelectMultiple(
            verbose_name=_('NixOS Modules'),
            is_stacked=False
        )
    )
    home_modules = forms.ModelMultipleChoiceField(
        queryset=generators.models.HomeManagerModule.objects.all(),
        required=False,
        widget=FilteredSelectMultiple(
            verbose_name=_('Home Manager Modules'),
            is_stacked=False
        )
    )

    def __init__(self, *args, **kwargs):
        super(RoleAdminForm, self).__init__(*args, **kwargs)

        if self.instance and self.instance.pk:
            self.fields['emacs_modules'].initial = self.instance.emacs_modules.all()
            self.fields['emacs_packages'].initial = self.instance.emacs_packages.all()
            self.fields['nixos_modules'].initial = self.instance.nixos_modules.all()
            self.fields['home_modules'].initial = self.instance.home_modules.all()

@admin.register(generators.models.GeneratorRole)
class RoleAdmin(admin.ModelAdmin):
    form = RoleAdminForm
#+end_src

* NEXT public site views to show information about the [[id:cce/cce][The Complete Computing Environment]] built around this arcology.

it would be nice to generate the tables of contents pages or subsections from the DB, for example...

#+begin_src python :tangle generators/views.py
from django.shortcuts import render

# Create your views here.
#+end_src

* The rest

#+begin_src python :tangle generators/apps.py
from django.apps import AppConfig


class GeneratorsConfig(AppConfig):
    default_auto_field = "django.db.models.BigAutoField"
    name = "generators"
#+end_src