arcology/generators.org

The Arroyo Generators

• Arroyo Generator Data Models
  • NEXT document the Generator pattern described by the Abstract Base Class.
    • NEXT Testing
  • Generate an Emacs configuration
    • NEXT Testing
  • Emacs custom package overrides
    • NEXT Testing
  • Declarative NixOS module imports
    • NEXT Testing
  • Declarative home-manager module imports
    • NEXT Testing
  • NEXT need to squash these
• NEXT Module tests
• Admin
• NEXT public site views to show information about the The Complete Computing Environment built around this arcology.
• The rest

talk about what these modules are used to do, show the management commands from the usage section again.

Arroyo Generator Data Models

NEXT document the Generator pattern described by the Abstract Base Class.

from __future__ import annotations
from typing import List
from django.db import models
from django.conf import settings
from django_prometheus.models import ExportModelOperationsMixin as EMOM

import arroyo.arroyo_rs as native

import roam.models

import pathlib
import graphlib

import logging
logger = logging.getLogger(__name__)


class GeneratorRole(EMOM('generatorRole'), models.Model):
    name = models.CharField(max_length=512, primary_key=True)

    @classmethod
    def get_or_create_many(cls, names: List[str]) -> List[GeneratorRole]:
        return [
            cls.objects.get_or_create(
                name=name
            )[0]
            for name in names
        ]


class Generator(EMOM('generator'), models.Model):
    class Meta:
        abstract = True

    ARROYO_KEYWORD_NAME = "THE_ABC"

    org_file = models.ForeignKey(
        roam.models.File,
        on_delete=models.CASCADE,
    )
    destination = models.CharField(max_length=512)

    roles = models.ManyToManyField(
        GeneratorRole,
        related_name="THE_ABC",
    )
    excluded_roles = models.ManyToManyField(
        GeneratorRole,
        related_name="THE_ABC",
    )

    def to_babel(self):
        raise NotImplemented

    def _to_babel_list(self, **kwargs):
        formatter = kwargs.get('formatter', "{}")
        return formatter.format(self.destination)

    def _to_babel_inclusion(self, **kwargs):
        formatter = kwargs.get('formatter', "{}")
        path = formatter.format(self.destination)
        with open(path, 'r') as f:
            return f.read()

    def __str__(self):
        return self.__repr__()

    def __repr__(self):
        return f"<{self.__class__.__name__}: {self.org_file.path} -> {self.destination}>"

    @classmethod
    def collect_for_babel(cls, **kwargs):
        return cls.objects.filter(roles__in=[kwargs['role']])

    @classmethod
    def create_from_arroyo(cls, doc: native.Document) -> List[Generator]:
        role_names = doc.collect_keywords("ARROYO_SYSTEM_ROLE") + doc.collect_keywords("ARROYO_ROLE")
        excluded_role_names = doc.collect_keywords("ARROYO_SYSTEM_EXCLUDE") + doc.collect_keywords("ARROYO_EXCLUDE_ROLE")
        f = roam.models.File.objects.get(path=doc.path)
        destinations = doc.collect_keywords(cls.ARROYO_KEYWORD_NAME)

        if len(destinations) == 0:
            # logger.debug(f"{cls} Skip {doc.path}")
            return []

        logger.debug(f"{cls} {doc.path} -> {destinations}")

        excluded_roles = GeneratorRole.get_or_create_many(excluded_role_names)
        if role_names == []:
            roles = [role for role in GeneratorRole.objects.all() if role not in excluded_roles]
        else:
            roles = GeneratorRole.get_or_create_many(role_names)

        ret = []
        for destination in destinations:
            obj = cls.objects.get_or_create(
                org_file=f,
                destination=destination,
            )[0]
            obj.roles.set(roles)
            obj.excluded_roles.set(excluded_roles)
            ret.append(obj)
        return ret

NEXT Testing

Test the classmethods create_from_arroyo and collect_for_babel and the to_babel instance method

role list and exclusions in create_from_arroyo

Generate an Emacs configuration

Emacs configuration is not declarative but imperative so dependency order needs to be established which the Nix modules do not nee to perform. #+ARROYO_MODULE_WANTS and #+ARROYO_MODULE_WANTED Keywords allow documents to describe dependency relationships. The python built-in graphlib">graphlib is used to order the EmacsSnippets, with the DependencyRelationship class caching edges between the snippets in the database.

class DependencyRelationship(EMOM('dependencyRelationship'), models.Model):
    # extract enough information to run https://docs.python.org/3/library/graphlib.html on the emacs snippets...
    dependency = models.ForeignKey(
        "EmacsSnippet",
        on_delete=models.CASCADE,
        blank=True,
        related_name="dependent_relations",
    )
    dependent = models.ForeignKey(
        "EmacsSnippet",
        on_delete=models.CASCADE,
        blank=True,
        related_name="dependancy_relations",
    )

    dependency_path = models.CharField(max_length=512)
    dependent_path = models.CharField(max_length=512)

    @classmethod
    def extract_ordering_from_arroyo(cls, ts: graphlib.TopologicalSorter, doc: native.Document):
        base_path_abs = pathlib.Path(settings.ARCOLOGY_BASE_DIR).expanduser()
        dependencies = doc.collect_keywords("ARROYO_MODULE_WANTS")
        dependents = doc.collect_keywords("ARROYO_MODULE_WANTED")

        ts.add(doc.path)

        for dep in dependencies:
            dep_path = str(base_path_abs.joinpath(dep))
            logger.debug(f"EXTRACT {doc.path} wants {dep_path}")
            ts.add(doc.path, dep_path)

        for dep in dependents:
            dep_path = str(base_path_abs.joinpath(dep))
            logger.debug(f"EXTRACT(rdep) {dep_path} wants {doc.path}")
            ts.add(doc.path, dep_path)

    @classmethod
    def collect_for_babel(cls) -> List[EmacsSnippet]:
        ts = graphlib.TopologicalSorter()
        mapping = dict()
        # add all the nodes from EmacsSnippet
        for snippet in EmacsSnippet.objects.all():
            path = snippet.org_file.path
            ts.add(path)
            mapping[path] = snippet
        # add all the DependencyRelationships
        for rel in cls.objects.all():
            pred = rel.dependency
            succ = rel.dependent
            ts.add(succ.org_file.path, pred.org_file.path)

        return list(map(lambda it: mapping[it], ts.static_order()))


class EmacsSnippet(EMOM('emacs_snippet'), Generator):
    ARROYO_KEYWORD_NAME = "ARROYO_EMACS_MODULE"

    roles = models.ManyToManyField(
        GeneratorRole,
        related_name="emacs_modules",
    )
    excluded_roles = models.ManyToManyField(
        GeneratorRole,
        related_name="excluded_emacs_modules",
    )

    dependencies = models.ManyToManyField(
        "EmacsSnippet",
        related_name="dependents",
        through="DependencyRelationship",
        through_fields=("dependency", "dependent"),
    )

    def to_babel(self) -> str:
        formatter = f"{settings.ARCOLOGY_EMACS_SNIPPETS_DIR}/{{}}.el"
        return self._to_babel_inclusion(formatter=formatter)

    @classmethod
    def collect_for_babel(cls, **kwargs):
        return DependencyRelationship.collect_for_babel()

    @classmethod
    def create_from_arroyo(cls, doc: native.Document) -> List[Generator]:
        """
        create_from_arroyo is overridden to persist ordering rules
        """
        new_snippets = super(EmacsSnippet, cls).create_from_arroyo(doc)
        base_path_abs = pathlib.Path(settings.ARCOLOGY_BASE_DIR).expanduser()

        # convert these to relation model objects..., but which?
        # back to the problem of defining relations to objects which may not yet exist...
        deps = doc.collect_keywords("ARROYO_MODULE_WANTS")
        reverse_deps = doc.collect_keywords("ARROYO_MODULE_WANTED")

        for obj in new_snippets:
            for dep in deps:
                dep_path = base_path_abs.joinpath(dep)
                logger.debug(f"create dep from {doc.path} to {dep_path}")
                the_dep = EmacsSnippet.objects.get(org_file__path=dep_path)

                DependencyRelationship.objects.create(
                    dependency=the_dep,
                    dependent=obj,
                )
            for dep in reverse_deps:
                dep_path = base_path_abs.joinpath(dep)
                logger.debug(f"create rdep to {dep_path} from {doc.path}")
                the_dep = EmacsSnippet.objects.get(org_file__path=dep_path)
                DependencyRelationship.objects.create(
                    dependency=obj,
                    dependent=the_dep,
                )

        return new_snippets

NEXT Testing

Test the classmethods and to_babel instance method

test the depdendencyrelationship stuff, the graphlib sort.

Emacs custom package overrides

It's possible for pages to stash Nix code snippets in to tangled files to make custom Emacs packages available to Arroyo Emacs using helpers like nixpkgs's melpaBuild and trivialBuild commands.

class EmacsEpkg(EMOM('emacs_epkg'), Generator):
    ARROYO_KEYWORD_NAME = "ARROYO_HOME_EPKGS"

    roles = models.ManyToManyField(
        GeneratorRole,
        related_name="emacs_packages",
    )
    excluded_roles = models.ManyToManyField(
        GeneratorRole,
        related_name="excluded_emacs_packages",
    )

    def to_babel(self) -> str:
        formatter = f"{settings.ARROYO_BASE_DIR}/{{}}"
        return self._to_babel_inclusion(formatter=formatter)

NEXT Testing

this one is pretty straightforward implementation of the ABC.

Declarative NixOS module imports

It's possible for pages to signal all or certain deployment roles to include a new NixOS module. This allows a user to download an org-mode file in to their org-roam directory and it will and new functionality to their computer.

class NixosModule(EMOM('nixos_module'), Generator):
    ARROYO_KEYWORD_NAME = "ARROYO_NIXOS_MODULE"

    roles = models.ManyToManyField(
        GeneratorRole,
        related_name="nixos_modules",
    )
    excluded_roles = models.ManyToManyField(
        GeneratorRole,
        related_name="excluded_nixos_modules",
    )

    def to_babel(self) -> str:
        # XXX this is going to need to be configurable at some point.....
        formatter = "../../{}"
        return self._to_babel_list(formatter=formatter)

NEXT Testing

this one is pretty straightforward implementation of the ABC.

Declarative home-manager module imports

It's possible to add home-manager modules to the user's profile in much the same way.

class HomeManagerModule(EMOM('home_manager_module'), Generator):
    ARROYO_KEYWORD_NAME = "ARROYO_HOME_MODULE"

    roles = models.ManyToManyField(
        GeneratorRole,
        related_name="home_modules",
    )
    excluded_roles = models.ManyToManyField(
        GeneratorRole,
        related_name="excluded_home_modules",
    )

    def to_babel(self) -> str:
        formatter = "{}"
        return self._to_babel_list(formatter=formatter)

NEXT Testing

this one is pretty straightforward implementation of the ABC.

NEXT need to squash these

# Generated by Django 4.2.6 on 2023-11-27 05:50

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):
    initial = True

    dependencies = [
        ("roam", "0005_alter_link_dest_heading"),
    ]

    operations = [
        migrations.CreateModel(
            name="GeneratorRole",
            fields=[
                (
                    "name",
                    models.CharField(max_length=512, primary_key=True, serialize=False),
                ),
            ],
        ),
        migrations.CreateModel(
            name="NixosModule",
            fields=[
                (
                    "id",
                    models.BigAutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("destination", models.CharField(max_length=512)),
                (
                    "excluded_roles",
                    models.ManyToManyField(
                        related_name="excluded_nixos_modules",
                        to="generators.generatorrole",
                    ),
                ),
                (
                    "org_file",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE, to="roam.file"
                    ),
                ),
                (
                    "roles",
                    models.ManyToManyField(
                        related_name="nixos_modules", to="generators.generatorrole"
                    ),
                ),
            ],
            options={
                "abstract": False,
            },
        ),
        migrations.CreateModel(
            name="HomeManagerModule",
            fields=[
                (
                    "id",
                    models.BigAutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("destination", models.CharField(max_length=512)),
                (
                    "excluded_roles",
                    models.ManyToManyField(
                        related_name="excluded_home_modules",
                        to="generators.generatorrole",
                    ),
                ),
                (
                    "org_file",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE, to="roam.file"
                    ),
                ),
                (
                    "roles",
                    models.ManyToManyField(
                        related_name="home_modules", to="generators.generatorrole"
                    ),
                ),
            ],
            options={
                "abstract": False,
            },
        ),
        migrations.CreateModel(
            name="EmacsSnippet",
            fields=[
                (
                    "id",
                    models.BigAutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("destination", models.CharField(max_length=512)),
                (
                    "excluded_roles",
                    models.ManyToManyField(
                        related_name="excluded_emacs_modules",
                        to="generators.generatorrole",
                    ),
                ),
                (
                    "org_file",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE, to="roam.file"
                    ),
                ),
                (
                    "roles",
                    models.ManyToManyField(
                        related_name="emacs_modules", to="generators.generatorrole"
                    ),
                ),
            ],
            options={
                "abstract": False,
            },
        ),
        migrations.CreateModel(
            name="EmacsEpkg",
            fields=[
                (
                    "id",
                    models.BigAutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("destination", models.CharField(max_length=512)),
                (
                    "excluded_roles",
                    models.ManyToManyField(
                        related_name="excluded_emacs_packages",
                        to="generators.generatorrole",
                    ),
                ),
                (
                    "org_file",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE, to="roam.file"
                    ),
                ),
                (
                    "roles",
                    models.ManyToManyField(
                        related_name="emacs_packages", to="generators.generatorrole"
                    ),
                ),
            ],
            options={
                "abstract": False,
            },
        ),
    ]

# Generated by Django 4.2.6 on 2023-12-11 20:11

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):
    dependencies = [
        ("generators", "0001_initial"),
    ]

    operations = [
        migrations.CreateModel(
            name="DependencyRelationship",
            fields=[
                (
                    "id",
                    models.BigAutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("dependency_path", models.CharField(max_length=512)),
                ("dependant_path", models.CharField(max_length=512)),
                (
                    "dependant",
                    models.ForeignKey(
                        blank=True,
                        on_delete=django.db.models.deletion.CASCADE,
                        related_name="dependancy_relations",
                        to="generators.emacssnippet",
                    ),
                ),
                (
                    "dependency",
                    models.ForeignKey(
                        blank=True,
                        on_delete=django.db.models.deletion.CASCADE,
                        related_name="dependant_relations",
                        to="generators.emacssnippet",
                    ),
                ),
            ],
        ),
        migrations.AddField(
            model_name="emacssnippet",
            name="dependencies",
            field=models.ManyToManyField(
                related_name="dependants",
                through="generators.DependencyRelationship",
                to="generators.emacssnippet",
            ),
        ),
    ]

# Generated by Django 4.2.6 on 2023-12-12 04:34

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):
    dependencies = [
        ("generators", "0002_dependencyrelationship_emacssnippet_dependencies"),
    ]

    operations = [
        migrations.RenameField(
            model_name="dependencyrelationship",
            old_name="dependant",
            new_name="dependent",
        ),
        migrations.RenameField(
            model_name="dependencyrelationship",
            old_name="dependant_path",
            new_name="dependent_path",
        ),
        migrations.AlterField(
            model_name="dependencyrelationship",
            name="dependency",
            field=models.ForeignKey(
                blank=True,
                on_delete=django.db.models.deletion.CASCADE,
                related_name="dependent_relations",
                to="generators.emacssnippet",
            ),
        ),
        migrations.AlterField(
            model_name="emacssnippet",
            name="dependencies",
            field=models.ManyToManyField(
                related_name="dependents",
                through="generators.DependencyRelationship",
                to="generators.emacssnippet",
            ),
        ),
    ]

NEXT Module tests

from django.test import TestCase

# Create your tests here.

Admin

from django.contrib import admin
from django import forms
from django.contrib.admin.widgets import FilteredSelectMultiple
from django.utils.translation import gettext as _

import generators.models


class DependencyInline(admin.TabularInline):
    model = generators.models.EmacsSnippet.dependencies.through
    fk_name = "dependent"

@admin.register(generators.models.EmacsEpkg)
@admin.register(generators.models.NixosModule)
@admin.register(generators.models.HomeManagerModule)
class GeneratorAdmin(admin.ModelAdmin):
    list_display = ["destination", "org_file"]
    filter_horizontal = ["excluded_roles", "roles"]

@admin.register(generators.models.EmacsSnippet)
class EmacsSnippetAdmin(admin.ModelAdmin):
    list_display = ["destination", "org_file"]
    filter_horizontal = ["excluded_roles", "roles"]
    inlines = [ DependencyInline ]

class RoleAdminForm(forms.ModelForm):
    emacs_modules = forms.ModelMultipleChoiceField(
        queryset=generators.models.EmacsSnippet.objects.all(),
        required=False,
        widget=FilteredSelectMultiple(
            verbose_name=_('Emacs Modules'),
            is_stacked=False
        )
    )
    emacs_packages = forms.ModelMultipleChoiceField(
        queryset=generators.models.EmacsEpkg.objects.all(),
        required=False,
        widget=FilteredSelectMultiple(
            verbose_name=_('Emacs Packages'),
            is_stacked=False
        )
    )
    nixos_modules = forms.ModelMultipleChoiceField(
        queryset=generators.models.NixosModule.objects.all(),
        required=False,
        widget=FilteredSelectMultiple(
            verbose_name=_('NixOS Modules'),
            is_stacked=False
        )
    )
    home_modules = forms.ModelMultipleChoiceField(
        queryset=generators.models.HomeManagerModule.objects.all(),
        required=False,
        widget=FilteredSelectMultiple(
            verbose_name=_('Home Manager Modules'),
            is_stacked=False
        )
    )

    def __init__(self, *args, **kwargs):
        super(RoleAdminForm, self).__init__(*args, **kwargs)

        if self.instance and self.instance.pk:
            self.fields['emacs_modules'].initial = self.instance.emacs_modules.all()
            self.fields['emacs_packages'].initial = self.instance.emacs_packages.all()
            self.fields['nixos_modules'].initial = self.instance.nixos_modules.all()
            self.fields['home_modules'].initial = self.instance.home_modules.all()

@admin.register(generators.models.GeneratorRole)
class RoleAdmin(admin.ModelAdmin):
    form = RoleAdminForm

NEXT public site views to show information about the The Complete Computing Environment built around this arcology.

it would be nice to generate the tables of contents pages or subsections from the DB, for example…

from django.shortcuts import render

# Create your views here.

The rest

from django.apps import AppConfig


class GeneratorsConfig(AppConfig):
    default_auto_field = "django.db.models.BigAutoField"
    name = "generators"