rrix
/
arroyo
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Compare commits

base: rrix:23c1e25b642adb087a6984381ece5f657859d37e
Branches Tags
rrix:main
rrix:rr/subheading-export
rrix:rr/rewrite-old
rrix:arroyo-rs
rrix:arroyo_rs_001
rrix:last-sqlmodel
...
compare: rrix:8b853694307976a8c8c8602baabc8b1bf15ff061
Branches Tags
rrix:main
rrix:rr/subheading-export
rrix:rr/rewrite-old
rrix:arroyo-rs
rrix:arroyo_rs_001
rrix:last-sqlmodel

2 Commits
23c1e25b64 ... 8b85369430

Author SHA1 Message Date
Ryan Rix 8b85369430 add greetz 2024-02-02 15:50:01 -08:00
Ryan Rix 104cc932d7 Arroyo is now reliant on the Arcology's commands, yikes! 2024-02-02 15:42:51 -08:00
10 changed files with 154 additions and 738 deletions
Show Stats Expand all files Collapse all files

105
arroyo-emacs.el
View File

@ -1,116 +1,15 @@
;; [[file:arroyo-emacs.org::*\[\[id:arroyo/system-cache\]\[Arroyo System Cache\]\] Emacs table][[[id:arroyo/system-cache][Arroyo System Cache]] Emacs table:1]]
(require 'arroyo-utils)
(defcustom arroyo-emacs-init-location (expand-file-name "~/arroyo-nix/files/init.el")
"Where `arroyo-emacs-generate-init' writes the amalgamated init file."
:group 'arroyo
:type 'string)
(dolist (kw '("ARROYO_EMACS_MODULE" "ARROYO_EMACS_TANGLED"))
(add-to-list 'arroyo-db-keywords kw nil #'equal))
(add-to-list 'arroyo-db--schemata
'(emacs
[(file :not-null)
(tangled :not-null)
(module :not-null)]))
;; [[id:arroyo/system-cache][Arroyo System Cache]] Emacs table:1 ends here
;; [[file:arroyo-emacs.org::*\[\[id:arroyo/system-cache\]\[Arroyo System Cache\]\] Emacs table][[[id:arroyo/system-cache][Arroyo System Cache]] Emacs table:2]]
(defun arroyo-emacs-update-db (&optional keywords-too)
(interactive "P")
(when keywords-too
(arroyo-db-update-all-roam-files))
(if (org-roam-file-p)
(arroyo-db-query [:delete :from emacs :where (= file $s1)]
(buffer-file-name))
(arroyo-db-query [:delete :from emacs]))
(when-let* ((module-args
(if (org-roam-file-p)
(list [:select [file "" value] :from keywords
:where (= keyword $s1) :and (= file $s2)]
"ARROYO_EMACS_MODULE"
(buffer-file-name))
(list [:select [file "" value] :from keywords
:where (= keyword $s1)]
"ARROYO_EMACS_MODULE")))
(modules (apply #'arroyo-db-query module-args)))
(arroyo-db-query [:insert :into emacs :values $v1]
(-map (pcase-lambda (`(,file ,_ ,module))
(vector file (arroyo-emacs--figure-tangled-location file) module))
modules))))
(defun arroyo-emacs--figure-tangled-location (file-path)
(->> file-path
(arroyo-db-query [:select value :from keywords
:where (= file $s1)
:and (= keyword "ARROYO_EMACS_MODULE")])
(caar)
(format "%s/cce/%s.el" org-roam-directory)))
(add-to-list 'arroyo-db-update-functions #'arroyo-emacs-update-db)
;; [[id:arroyo/system-cache][Arroyo System Cache]] Emacs table:2 ends here
;; [[file:arroyo-emacs.org::*Arroyo Emacs =init.el= generator][Arroyo Emacs =init.el= generator:1]]
(defun arroyo-emacs-generate-init ()
(interactive)
;; (setenv "ARCOLOGY_DB_PATH" "/home/rrix/org/arcology-django/db.sqlite3")
;; (shell-command-to-string "nix run path:/home/rrix/org/arcology-django#arcology -- ingest_files ~/org &>/dev/null")
;; (shell-command-to-string (format "nix run path:/home/rrix/org/arcology-django#arcology -- generate -m emacs -r server -d %s" arroyo-emacs-init-location))
(arroyo-generate-imports "emacs" nil arroyo-emacs-init-location)
(find-file arroyo-emacs-init-location))
;; (defun arroyo-emacs-generate-init ()
;; "Write an emacs `init.el' to `arroyo-emacs-init-location' using
;; tangled files spread throughout an org-roam wiki."
;; (interactive)
;; (arroyo-emacs-update-db)
;; (let ((dest arroyo-emacs-init-location)
;; (files (arroyo-emacs--ordered-files)))
;; (with-current-buffer (find-file-noselect dest)
;; (erase-buffer)
;; (dolist (file files)
;; (insert-file-contents file)
;; (goto-char (point-max)))
;; (save-buffer))))
(defun arroyo-emacs--module-pair-for-rel-file (unordered rel-file)
(second (-find (lambda (elt)
(equal (expand-file-name rel-file org-roam-directory)
(car elt)))
unordered)))
(defun arroyo-emacs--ordered-files () ;; (ref:ordered-files)
(let* ((unordered (arroyo-db-query [:select [file tangled] :from emacs])) ;; (ref:unordered)
(unordered-filenames (-map #'car unordered)) ;; (ref:unordered-filenames)
(order (arroyo-utils-order-modules unordered-filenames)) ;; (ref:order-modules)
(partialfn (apply-partially #'arroyo-emacs--module-pair-for-rel-file unordered))
(ordered (-map partialfn order)))
(->>
ordered
(-filter #'identity)
(-uniq))))
;; Arroyo Emacs =init.el= generator:1 ends here
;; [[file:arroyo-emacs.org::*\[\[id:cce/literate_programming\]\[Literate Programming\]\] helpers][[[id:cce/literate_programming][Literate Programming]] helpers:1]]
(defun arroyo-emacs-epkgs ()
(->> (arroyo-db-query [:select [file value] :from keywords
:where (= keyword "ARROYO_HOME_EPKGS")])
(-map #'second)
(-map (lambda (path)
(let ((path (expand-file-name path "~/arroyo-nix")))
(with-current-buffer (find-file-noselect path)
(let ((bs (buffer-string)))
(kill-this-buffer)
bs)))))
(s-join "\n")))
(add-to-list 'arroyo-db-keywords "ARROYO_HOME_EPKGS" nil #'equal)
(defun arroyo-epkg-overrides (&optional role)
(arroyo-generate-imports "epkgs" role nil nil))
;; [[id:cce/literate_programming][Literate Programming]] helpers:1 ends here
;; [[file:arroyo-emacs.org::generate_epkg_overrides][generate_epkg_overrides]]
(arroyo-emacs-epkgs)
;; generate_epkg_overrides ends here
;; [[file:arroyo-emacs.org::*need a reverse of =ARROYO_MODULE_WANTS= for specifying "wanted-by"s][need a reverse of =ARROYO_MODULE_WANTS= for specifying "wanted-by"s:1]]
(provide 'arroyo-emacs)
;; need a reverse of =ARROYO_MODULE_WANTS= for specifying "wanted-by"s:1 ends here

160
arroyo-emacs.org
View File

@ -21,61 +21,6 @@ Users of the Arroyo Home Manager generator will have access to features availabl
- Native GTK with Wayland support (though Wayland is not supported by impotant parts of Arroyo like the [[id:cce/exwm][EXWM]] system)
- When using =use-package=, packages will be downloaded and (natively!) compiled in the deploy phase rather than on Emacs startup
* [[id:arroyo/system-cache][Arroyo System Cache]] Emacs table
#+begin_src emacs-lisp :results none
(require 'arroyo-utils)
(defcustom arroyo-emacs-init-location (expand-file-name "~/arroyo-nix/files/init.el")
"Where `arroyo-emacs-generate-init' writes the amalgamated init file."
:group 'arroyo
:type 'string)
(dolist (kw '("ARROYO_EMACS_MODULE" "ARROYO_EMACS_TANGLED"))
(add-to-list 'arroyo-db-keywords kw nil #'equal))
(add-to-list 'arroyo-db--schemata
'(emacs
[(file :not-null)
(tangled :not-null)
(module :not-null)]))
#+end_src
#+begin_src emacs-lisp :results none
(defun arroyo-emacs-update-db (&optional keywords-too)
(interactive "P")
(when keywords-too
(arroyo-db-update-all-roam-files))
(if (org-roam-file-p)
(arroyo-db-query [:delete :from emacs :where (= file $s1)]
(buffer-file-name))
(arroyo-db-query [:delete :from emacs]))
(when-let* ((module-args
(if (org-roam-file-p)
(list [:select [file "" value] :from keywords
:where (= keyword $s1) :and (= file $s2)]
"ARROYO_EMACS_MODULE"
(buffer-file-name))
(list [:select [file "" value] :from keywords
:where (= keyword $s1)]
"ARROYO_EMACS_MODULE")))
(modules (apply #'arroyo-db-query module-args)))
(arroyo-db-query [:insert :into emacs :values $v1]
(-map (pcase-lambda (`(,file ,_ ,module))
(vector file (arroyo-emacs--figure-tangled-location file) module))
modules))))
(defun arroyo-emacs--figure-tangled-location (file-path)
(->> file-path
(arroyo-db-query [:select value :from keywords
:where (= file $s1)
:and (= keyword "ARROYO_EMACS_MODULE")])
(caar)
(format "%s/cce/%s.el" org-roam-directory)))
(add-to-list 'arroyo-db-update-functions #'arroyo-emacs-update-db)
#+end_src
* Arroyo Emacs =init.el= generator
:PROPERTIES:
:ID: arroyo/emacs/init.el
@ -83,57 +28,18 @@ Users of the Arroyo Home Manager generator will have access to features availabl
=arroyo-emacs-generate-init= can be called interactively[fn:1:that is, by pressing =Alt-x= / =M-x= and then typing the name of the command; [[id:modern_interface_terms][Modern Interface Terms]]] to assemble an =init.el= file which will have the dependencies ordered in such a way as to allow the user to user Emacs.
#+ARROYO_MODULE_WANTS: arcology-django/interfaces.org
#+begin_src emacs-lisp :results none -r
(defun arroyo-emacs-generate-init ()
(interactive)
;; (setenv "ARCOLOGY_DB_PATH" "/home/rrix/org/arcology-django/db.sqlite3")
;; (shell-command-to-string "nix run path:/home/rrix/org/arcology-django#arcology -- ingest_files ~/org &>/dev/null")
;; (shell-command-to-string (format "nix run path:/home/rrix/org/arcology-django#arcology -- generate -m emacs -r server -d %s" arroyo-emacs-init-location))
(arroyo-generate-imports "emacs" nil arroyo-emacs-init-location)
(find-file arroyo-emacs-init-location))
;; (defun arroyo-emacs-generate-init ()
;; "Write an emacs `init.el' to `arroyo-emacs-init-location' using
;; tangled files spread throughout an org-roam wiki."
;; (interactive)
;; (arroyo-emacs-update-db)
;; (let ((dest arroyo-emacs-init-location)
;; (files (arroyo-emacs--ordered-files)))
;; (with-current-buffer (find-file-noselect dest)
;; (erase-buffer)
;; (dolist (file files)
;; (insert-file-contents file)
;; (goto-char (point-max)))
;; (save-buffer))))
(defun arroyo-emacs--module-pair-for-rel-file (unordered rel-file)
(second (-find (lambda (elt)
(equal (expand-file-name rel-file org-roam-directory)
(car elt)))
unordered)))
(defun arroyo-emacs--ordered-files () ;; (ref:ordered-files)
(let* ((unordered (arroyo-db-query [:select [file tangled] :from emacs])) ;; (ref:unordered)
(unordered-filenames (-map #'car unordered)) ;; (ref:unordered-filenames)
(order (arroyo-utils-order-modules unordered-filenames)) ;; (ref:order-modules)
(partialfn (apply-partially #'arroyo-emacs--module-pair-for-rel-file unordered))
(ordered (-map partialfn order)))
(->>
ordered
(-filter #'identity)
(-uniq))))
#+end_src
This [[(ordered-files)]] function is a little bit dense to read, so let's take this apart let-binding by let-binding:
- [[(unordered)]] takes the modules out of the [[id:arroyo/system-cache][Arroyo System Cache]] as pairs of [file tangled-file location] and then [[(ref:unordered-filenames)]] extracts the first filename from the unordered pairs.
- [[(ref:order-modules)]] orders the unordered file-names based on a database-backed topological sort implemented in [[id:4d62fe38-8611-4794-b2cf-b9900ff3294b][Arroyo Dependency Sorting]] in =arroyo-utils=.
- [[(module-pair-for-rel-file)]] is a function designed to find a result-pair from the list of [[(unordered)]] which matches the file-name in the newly-reordered fashion. I'm sure there's a better way to do this than with a map+find operation, I failed the technical interview.
In total, this takes the unordered dataset and returns them with a topological sort applied to them, the edges of the topology are pulled from the [[id:4d62fe38-8611-4794-b2cf-b9900ff3294b][Arroyo Dependency Sorting]] cache.
The interactive function is more or less as it says on the tin -- it uses the Arcology stuff to write the =init.el= to a local directory for the [[id:20211130T215142.470274][arroyo-flood]].
The interactive function is more or less as it says on the tin -- it writes the =init.el= to a local directory for the [[id:20211130T215142.470274][arroyo-flood]].
* [[id:arroyo/home-manager][Arroyo Home Manager]] support for Emacs
* [[id:arroyo/home-manager][Arroyo Home Manager]] support for customized Emacs package
:PROPERTIES:
:ID: 3018859a-940b-4096-a083-133f361e312c
:ROAM_ALIASES: "Emacs Nix Package"
@ -247,6 +153,30 @@ Enable [[roam:ccache]] for =emacsGcc= builds in [[id:cce/my_nixos_configuration]
}
#+end_src
** [[id:cce/literate_programming][Literate Programming]] helpers
I want to be able to construct a [[id:cce/home-manager][home-manager]] import dynamically which can be used to install with =emacs-overlay= functionality. I also want it to be able to inject arbitrary =packageOverrides= in to it via references in stored in documents' =ARROYO_HOME_EPKGS= property keyword.
#+begin_src emacs-lisp :results none
(defun arroyo-epkg-overrides (&optional role)
(arroyo-generate-imports "epkgs" role nil nil))
#+end_src
=arroyo-epkg-overrides= can be used to insert any overridden elisp package in to the registry for =use-package= to install, including things with patches or personal forks. yummy. apply a =ARROYO_HOME_EPKGS= [[id:e050ed03-044f-4a48-912d-72579bef9a26][keyword]] to the page to include it here.
#+NAME: generate_epkg_overrides
#+begin_src emacs-lisp :tangle nil
(arroyo-epkg-overrides)
#+end_src
=arroyo-emacs-init-location= above is simply the [[id:arroyo/emacs][Arroyo Emacs]]-managed =init.el=:
#+NAME: arroyo-emacs-init-location
#+begin_src emacs-lisp :tangle no
; arroyo-emacs-init-location
"<arroyo/files/init.el>"
#+end_src
** NEXT fix ccache support
** DONE reenable ccache
SCHEDULED: <2021-09-22 Wed>
@ -258,40 +188,6 @@ SCHEDULED: <2021-09-22 Wed>
https://github.com/NixOS/nixpkgs/pull/137936
** [[id:cce/literate_programming][Literate Programming]] helpers
I want to be able to construct a [[id:cce/home-manager][home-manager]] import dynamically which can be used to install with =emacs-overlay= functionality. I also want it to be able to inject arbitrary =packageOverrides= in to it via references in stored in documents' =ARROYO_HOME_EPKGS= property keyword.
#+begin_src emacs-lisp :results none
(defun arroyo-emacs-epkgs ()
(->> (arroyo-db-query [:select [file value] :from keywords
:where (= keyword "ARROYO_HOME_EPKGS")])
(-map #'second)
(-map (lambda (path)
(let ((path (expand-file-name path "~/arroyo-nix")))
(with-current-buffer (find-file-noselect path)
(let ((bs (buffer-string)))
(kill-this-buffer)
bs)))))
(s-join "\n")))
(add-to-list 'arroyo-db-keywords "ARROYO_HOME_EPKGS" nil #'equal)
#+end_src
=arroyo-emacs-epkgs= can be used to insert any overridden elisp package in to the registry for =use-package= to install, including things with patches or personal forks. yummy. apply a =ARROYO_HOME_EPKGS= [[id:e050ed03-044f-4a48-912d-72579bef9a26][keyword]] to the page to include it here.
#+NAME: generate_epkg_overrides
#+begin_src emacs-lisp
(arroyo-emacs-epkgs)
#+end_src
=arroyo-emacs-init-location= above is simply the [[id:arroyo/emacs][Arroyo Emacs]]-managed =init.el=:
#+NAME: arroyo-emacs-init-location
#+begin_src emacs-lisp :tangle no
; arroyo-emacs-init-location
"<arroyo/files/init.el>"
#+end_src
** NEXT how to automate epkg tangling, etc.
can use [[id:cce/org-roam][org-roam]]'s [[id:20211203T142617.812313][Arcology.Roam.File]] to know whether a file has been updated and tangle it as part of the dynamic-home-manager dynamic-nixops etc... those things will check a cache to see if the files have changed, and tangle them and then this file if so?

2
arroyo-feed-cache.org
View File

@ -8,6 +8,8 @@
#+ARCOLOGY_KEY: cce/arroyo/feeds
#+ARCOLOGY_ALLOW_CRAWL: t
*this isn't being used right now, sorry.*
Arroyo Feed Cache is used to generate a list of RSS and Atom URLs from across a [[id:cce/org-roam][org-roam]] [[id:knowledge_base][Knowledge Base]] in a format which can be used by the [[https://github.com/sloonz/ua][Universal Aggregator]] to present a =Maildir= cache of the feeds' contents so that they can be viewed in [[id:cce/gnus][Gnus]]. [[id:cce/universal_aggregator][Universal Aggregator]] is a [[https://xkcd.com/2347/][small but important]] part of my [[id:26c9e4fd-4501-4b8b-95ce-a2a5230d7c1e][Email and News and Information Pipelines]], but it's fairly simple and quite robust. I'll have documentation for operating UA itself defined elsewhere, Arroyo Feed Generator puts the =ggsrc= file in place.
It is managed and updated and curated in fits and bursts and when I want to expand my news horizon in some fashion. see [[id:79db5d1e-b52e-41e2-b75c-8a7c5c5a05c9][YouTube Feeds]] as an example in use; automatically generate from a table, copy to server, restart =UA= with [[id:arroyo/feed-cache-ggsrc][=arroyo-feeds-flood=]].

89
arroyo-home-manager.el
View File

@ -1,87 +1,8 @@
(add-to-list 'arroyo-db-keywords "ARROYO_HOME_MODULE" nil #'equal)
(add-to-list 'arroyo-db-keywords "CCE_HOME_MODULE" nil #'equal)
(add-to-list 'arroyo-db--schemata
'(home-manager
[(file :not-null)
;(file-hash :not-null)
(home-module :not-null)
role]))
(defun arroyo-home-manager-update-db (&optional keywords-too)
(interactive "P")
(when keywords-too
(arroyo-db-update-all-roam-files))
(if (org-roam-file-p)
(arroyo-db-query [:delete :from home_manager :where (= file $s1)]
(buffer-file-name))
(arroyo-db-query [:delete :from home_manager]))
(when-let* ((args
(if (org-roam-file-p)
(list [:select [file value] :from keywords
:where (in keyword $v1) :and (= file $s2)]
(vector "ARROYO_HOME_MODULE" "CCE_HOME_MODULE")
(buffer-file-name))
(list [:select [file value] :from keywords
:where (in keyword $v1)]
(vector "ARROYO_HOME_MODULE" "CCE_HOME_MODULE"))))
(results (apply #'arroyo-db-query args))
;; attach role
(results (-map (pcase-lambda (`(,file ,value))
(list file value (arroyo-db-get "ARROYO_SYSTEM_ROLE" file)))
results)))
(arroyo-db-query [:insert :into home-manager :values $v1]
(-map (lambda (result) (apply #'vector result))
results))))
(add-to-list 'arroyo-db-update-functions #'arroyo-home-manager-update-db)
(defun arroyo-home-manager-imports1 ()
(->> [:select [home-module file] :from home-manager]
(arroyo-db-query)))
;; [[file:arroyo-home-manager.org::arroyo-home-manager-imports][arroyo-home-manager-imports]]
(defun arroyo-home-manager-imports (&optional role)
(setenv "ARCOLOGY_DB_PATH" "/home/rrix/org/arcology-django/db.sqlite3")
(shell-command-to-string "nix run path:/home/rrix/org/arcology-django#arcology -- ingest_files ~/org &>/dev/null")
(shell-command-to-string (s-join " " `("nix run path:/home/rrix/org/arcology-django#arcology"
"--"
"generate -m home-manager"
,(when role
(format "-r %s" role))
"2>/dev/null"
" | sort"))))
;; (defun *arroyo-home-manager-imports (&optional role)
;; (if-let* ((imports (arroyo-home-manager-imports1))
;; (no-role? (not role)))
;; (->> imports
;; (-map #'car)
;; (-sort #'s-less?)
;; (s-join "\n"))
;; (progn
;; (let* ((all-files (-map #'cadr imports))
;; (role-only-files (-map #'car (arroyo-db-query [:select file :from home-manager :where (like role $r1)] (format "%%%s%%" role))))
;; (exclude-these-files (arroyo-db-by-keyword "ARROYO_SYSTEM_EXCLUDE" role))
;; ;; also need to exclude things whose ARROYO_SYSTEM_ROLE is not `role'!
;; (not-my-role-files
;; (-map #'car
;; (arroyo-db-query [:select file :from home-manager
;; :where (!= role $s1)]
;; (list role))))
;; ;; combine all those lists in to a single authoritative list.
;; (amalgamated-files
;; (-union role-only-files
;; (->> all-files
;; (--remove (-contains? exclude-these-files it))
;; (--remove (-contains? not-my-role-files it)))))
;; ;; make them relative to org-roam-directory for the module orderer
;; (rel-files (--map (file-relative-name it org-roam-directory) amalgamated-files)))
;; ;; filter imports
;; (->> rel-files
;; (arroyo-utils-order-modules)
;; (-map (lambda (file)
;; (caar (arroyo-db-query [:select home-module :from home-manager
;; :where (= file $s1)]
;; (expand-file-name file org-roam-directory)))))
;; (-filter #'identity)
;; (-sort #'s-less?))))))
(arroyo-generate-imports "home-manager" role))
;; arroyo-home-manager-imports ends here
;; [[file:arroyo-home-manager.org::+begin_src emacs-lisp][No heading:2]]
(provide 'arroyo-home-manager)
;; No heading:2 ends here

93
arroyo-home-manager.org
View File

@ -6,107 +6,18 @@
#+ARCOLOGY_KEY: cce/arroyo/home-manager
#+ARCOLOGY_ALLOW_CRAWL: t
Arroyo can construct a configuration for [[https://github.com/nix-community/home-manager][nix-community/home-manager]] from any number of [[id:cce/org-roam][org-roam]] files using the [[id:arroyo/system-cache][Arroyo System Cache]] and [[id:cce/literate_programming][Org Babel]]. There is a colloquial "nixlib" directory, currently [[file:~/org/cce/nixlib]] which all [[id:c75d20e6-8888-4c5a-ac97-5997e2f1c711][NixOS]] files are relative to. By adding a keyword =#+ARROYO_HOME_MODULE: hm/import-file.nix= and then tangling to e.g. =~/org/cce/nixlib/hm/import-file.nix= you can generate a modular [[id:cce/home-manager][home-manager configuration]].
Arroyo can construct a configuration for [[https://github.com/nix-community/home-manager][nix-community/home-manager]] from any number of [[id:cce/org-roam][org-roam]] files using the [[id:arroyo/system-cache][Arroyo System Cache]] and [[id:cce/literate_programming][Org Babel]]. There is a colloquial "nixlib" directory, currently [[file:~/arroyo-nix/]] which all [[id:c75d20e6-8888-4c5a-ac97-5997e2f1c711][Nix]] files are relative to. By adding a keyword =#+ARROYO_HOME_MODULE: hm/import-file.nix= and then tangling to e.g. =~/nix/hm/import-file.nix= you can generate a modular [[id:cce/home-manager][home-manager configuration]] like I do.
#+PROPERTY: header-args :mkdirp yes
#+PROPERTY: header-args:emacs-lisp :tangle arroyo-home-manager.el :results none
#+AUTO_TANGLE: t
: "ARROYO_HOME_EPKGS" used in emacs generator
#+begin_src emacs-lisp
(add-to-list 'arroyo-db-keywords "ARROYO_HOME_MODULE" nil #'equal)
(add-to-list 'arroyo-db-keywords "CCE_HOME_MODULE" nil #'equal)
(add-to-list 'arroyo-db--schemata
'(home-manager
[(file :not-null)
;(file-hash :not-null)
(home-module :not-null)
role]))
#+end_src
#+begin_src emacs-lisp
(defun arroyo-home-manager-update-db (&optional keywords-too)
(interactive "P")
(when keywords-too
(arroyo-db-update-all-roam-files))
(if (org-roam-file-p)
(arroyo-db-query [:delete :from home_manager :where (= file $s1)]
(buffer-file-name))
(arroyo-db-query [:delete :from home_manager]))
(when-let* ((args
(if (org-roam-file-p)
(list [:select [file value] :from keywords
:where (in keyword $v1) :and (= file $s2)]
(vector "ARROYO_HOME_MODULE" "CCE_HOME_MODULE")
(buffer-file-name))
(list [:select [file value] :from keywords
:where (in keyword $v1)]
(vector "ARROYO_HOME_MODULE" "CCE_HOME_MODULE"))))
(results (apply #'arroyo-db-query args))
;; attach role
(results (-map (pcase-lambda (`(,file ,value))
(list file value (arroyo-db-get "ARROYO_SYSTEM_ROLE" file)))
results)))
(arroyo-db-query [:insert :into home-manager :values $v1]
(-map (lambda (result) (apply #'vector result))
results))))
(add-to-list 'arroyo-db-update-functions #'arroyo-home-manager-update-db)
#+end_src
=arroyo-home-manager-imports= returns a string suitable for =import='ing in [[id:f32b616e-ede6-461e-8bae-b5f1c717a05e][home-manager's literate helpers]].
#+NAME: arroyo-home-manager-imports
#+begin_src emacs-lisp
(defun arroyo-home-manager-imports1 ()
(->> [:select [home-module file] :from home-manager]
(arroyo-db-query)))
(defun arroyo-home-manager-imports (&optional role)
(setenv "ARCOLOGY_DB_PATH" "/home/rrix/org/arcology-django/db.sqlite3")
(shell-command-to-string "nix run path:/home/rrix/org/arcology-django#arcology -- ingest_files ~/org &>/dev/null")
(shell-command-to-string (s-join " " `("nix run path:/home/rrix/org/arcology-django#arcology"
"--"
"generate -m home-manager"
,(when role
(format "-r %s" role))
"2>/dev/null"
" | sort"))))
;; (defun *arroyo-home-manager-imports (&optional role)
;; (if-let* ((imports (arroyo-home-manager-imports1))
;; (no-role? (not role)))
;; (->> imports
;; (-map #'car)
;; (-sort #'s-less?)
;; (s-join "\n"))
;; (progn
;; (let* ((all-files (-map #'cadr imports))
;; (role-only-files (-map #'car (arroyo-db-query [:select file :from home-manager :where (like role $r1)] (format "%%%s%%" role))))
;; (exclude-these-files (arroyo-db-by-keyword "ARROYO_SYSTEM_EXCLUDE" role))
;; ;; also need to exclude things whose ARROYO_SYSTEM_ROLE is not `role'!
;; (not-my-role-files
;; (-map #'car
;; (arroyo-db-query [:select file :from home-manager
;; :where (!= role $s1)]
;; (list role))))
;; ;; combine all those lists in to a single authoritative list.
;; (amalgamated-files
;; (-union role-only-files
;; (->> all-files
;; (--remove (-contains? exclude-these-files it))
;; (--remove (-contains? not-my-role-files it)))))
;; ;; make them relative to org-roam-directory for the module orderer
;; (rel-files (--map (file-relative-name it org-roam-directory) amalgamated-files)))
;; ;; filter imports
;; (->> rel-files
;; (arroyo-utils-order-modules)
;; (-map (lambda (file)
;; (caar (arroyo-db-query [:select home-module :from home-manager
;; :where (= file $s1)]
;; (expand-file-name file org-roam-directory)))))
;; (-filter #'identity)
;; (-sort #'s-less?))))))
(arroyo-generate-imports "home-manager" role))
#+end_src
#+begin_src emacs-lisp

180
arroyo-native-parser.org
View File

@ -217,9 +217,7 @@ Eventually this will include an app that will generate the DB, and commands whic
edition = "2021"
#+end_src
* Rust package :Code:
** The Parser Document Types
* The Parser Document Types
:PROPERTIES:
:header-args:rust: :tangle src/types.rs :mkdirp yes
:ID: 20231024T105239.712407
@ -248,7 +246,7 @@ The parser is a big state machine that iterates over every node in the document
- information about the link destination
- text of the link
*** Document
** Document
A document contains a list of headings, and a list of keywords.
@ -307,7 +305,7 @@ impl Document {
}
#+end_src
*** Keyword
** Keyword
A =Keyword= is extracted from =#+KEYWORD: value= syntax keywords in the document. These three-value tuples of =[file, keyword, value]= are where metadata used by the Arroyo generators is defined.
@ -345,7 +343,7 @@ impl Keyword {
}
#+end_src
*** Heading
** Heading
The fields in =Heading= are defined based on what is available in [[id:cce/org-roam][org-roam]]; These are the basic information used by the [[id:arcology/arroyo-page][Arroyo Arcology Generator]].
@ -399,7 +397,7 @@ impl Heading {
}
#+end_src
*** Link
** Link
A link knows where it is because it knows where it isn't. [[https://www.youtube.com/watch?v=6iBeRfOAAwk][Just kidding]], they have information about where they were and where they're going and the text inside the link.
@ -444,7 +442,7 @@ impl Link {
}
#+end_src
** The Parser
* The Arroyo Org Parser
:PROPERTIES:
:header-args:rust: :tangle src/parse.rs :mkdirp yes
:ID: 20240112T120658.817314
@ -466,7 +464,7 @@ use std::{error::Error, fs};
use crate::types::{Document, Heading, InvalidDocError, Keyword, Link};
#+end_src
*** The public interface
** The public interface
The public interface parses the document in to an AST and then returns a Document, defined above, populated with the code below in [[id:20231023T123818.667030][Extracting Arroyo Keywords]] and [[id:20231023T124047.857348][Extracting Arroyo Headings]].
@ -493,7 +491,7 @@ pub fn parse_document(path: String) -> Result<Document> {
}
#+end_src
*** Extracting Arroyo Keywords
** Extracting Arroyo Keywords
:PROPERTIES:
:ID: 20231023T123818.667030
:END:
@ -513,7 +511,7 @@ pub fn extract_metadata(path: String, tree: &Org) -> Result<Vec<Keyword>> {
}
#+end_src
*** Extracting Arroyo Headings
** Extracting Arroyo Headings
:PROPERTIES:
:ID: 20231023T124047.857348
:END:
@ -587,7 +585,7 @@ tree.iter()
match event {
#+END_SRC
**** Heading parser
*** Heading parser
=orgize::Element::Title= is a heading. When the parser encounters a new heading it:
- updates the =cur_id= and =cur_level= values, update the ID "breadcrumbs"; links use this to "wrap up" to the first parent with an ID.
@ -642,12 +640,12 @@ Event::Start(orgize::Element::Title(title)) => {
}
#+END_SRC
***** NEXT I should be doing something like the =inherited_tags= stuff to track =cur_id= inheritance...
**** NEXT I should be doing something like the =inherited_tags= stuff to track =cur_id= inheritance...
:PROPERTIES:
:ID: 20231023T133016.903009
:END:
**** File-level Property Drawer parsing
*** File-level Property Drawer parsing
:PROPERTIES:
:ID: 20240116T235328.441922
:END:
@ -719,9 +717,9 @@ Event::End(orgize::Element::Drawer(_drawer)) => {
}
#+END_SRC
***** NEXT fix orgize to expose file-level propertiesmap
**** NEXT fix orgize to expose file-level propertiesmap
**** Link parsing
*** Link parsing
Look; I'm gonna be honest here. I don't remember why the links are stored outside the heading until the end of the document parsing. Some ownership bullshit, and the COW types, if I recall.
@ -764,8 +762,8 @@ Event::Start(orgize::Element::Link(link)) => {
}
#+END_SRC
**** NEXT Attachment and image caching
**** Cleaning up
*** NEXT Attachment and image caching
*** Cleaning up
Having populated all these variables, the headings have the links spliced back in to them and returned.
@ -788,7 +786,7 @@ Having populated all these variables, the headings have the links spliced back i
}
#+end_src
*** =split_quoted_string=
** =split_quoted_string=
:PROPERTIES:
:ID: 20231023T130916.139809
:END:
@ -832,20 +830,21 @@ fn split_quoted_string(quoted_str: String) -> Result<Vec<String>, Box<dyn Error>
I wrote some simple unit tests for this below.
** The HTML exporter
* The Arroyo HTML exporter
:PROPERTIES:
:header-args:rust: :tangle src/export_html.rs :mkdirp yes
:ID: 20240112T120813.386800
:END:
I'm not exactly looking forward to writing this lul.
This thing is a little bit complex, so I hope I've documented it well enough. In short, the feature-set of this HTML exporter is designed to match the requirements of [[id:arcology/django/readme][The Arcology Project: Django Edition]]:
- rewrite URLs from a HashMap passed in; higher-level things can map from IDs to URLs, files to URLs, and re-write the rest to 404s.
- re-write [[id:2e31b385-a003-4369-a136-c6b78c0917e1][org-fc]] clozes to be =<span>='s
- re-write [[id:2e31b385-a003-4369-a136-c6b78c0917e1][org-fc]] clozes to be =<span>='s (currently supported except for clozes with links in them)
- drop org-fc drawers
- code highlighting by building this on top of =Orgize::export_html::SyntectHtmlHandler=
stretch:
stretch unimplemented:
- tufte side-notes
=crate::export_html::htmlize_file= is the public entrypoint for this functionality, below.
@ -941,9 +940,9 @@ impl<E: From<Error>, H: HtmlHandler<E>> Default for ArroyoHtmlHandler<E, H> {
}
#+end_src
*** The Custom HTML Exporter Extensions
** The Custom HTML Exporter Extensions
=HtmlHandler='s need to implement a =start= and =end= function which take elements to act on, sort of like how the iterator for the parser works. We just go matching for the elements that are important and let the rest fall through to the "inner" handler.
=ArroyoHtmlHandler= need to implement =HtmlHandler='s =start= and =end= function which take elements to act on, sort of like how the iterator for the parser works. We just go matching for the elements that are important and let the rest fall through to the "inner" handler.
#+begin_src rust
impl<E: From<Error>, H: HtmlHandler<E>> HtmlHandler<E> for ArroyoHtmlHandler<E, H> {
@ -1039,11 +1038,9 @@ Everything else is passed along to Syntect or the default HTML Handler.
And that's how we implement the export features. Calling it from Rust is pretty straightforward, and from Python it's even easier.
*** The Public Interface
** The API Interface
Unfortunately the =parse_custom= invocation is repeated between both of these entrypoints because I'm not a good enough rust programmer to figure out the lifecycle of some strings and whatnot that get embedded in the =orgize::Org= object.
The exporter embeds an exporter with a code formatting pass applied to it, I'd like to be able to pass a configuration to that some day.
The exporter embeds another exporter with a code formatting pass applied to it, I'd like to be able to pass a configuration to that some day...
#+begin_src rust
// sure would be nice..... some day i'll understand lifetimes enough
@ -1075,10 +1072,21 @@ pub fn htmlize_file(path: String, options: ExportOptions) -> Result<String> {
}
#+end_src
** NEXT The Atom exporter
Unfortunately the =parse_custom= invocation is repeated between many of these entrypoints, the Atom and HTML and parser, because I'm not a good enough rust programmer yet to figure out the lifecycle of some strings and whatnot that get embedded in the =orgize::Org= object.
** NEXT organize this thing's module hierarchy a bit
why no
export::html -> export::document
export::atom -> export::heading
* INPROGRESS The Atom exporter
:PROPERTIES:
:ID: 20240112T121157.583809
:END:
:LOGBOOK:
- State "INPROGRESS" from "NEXT" [2024-02-02 Fri 14:41]
:END:
The HTML parser turns an org mode document in to HTML.
@ -1088,7 +1096,7 @@ For now maybe it is easier to assume that the headings are all in one file; that
the primary tension of the arroyo library now is that its design context is only in the realm of the arcology project's design goals. I need to start deciding whether a design goal of this library is to support non-arcology document systems. surely interoperable but different document systems could be built on top of arroyo
*** First Pass
** First Pass
so the first pass of this API could take a file path, extract the feed metadata from keywords and heading properties; it could construct an entire atom feed, falling back to the custom HTML exporter to fill out the feed with text content. That's probably fine, and an API that other document servers could work with.
@ -1483,7 +1491,7 @@ Processing a title in to an Atom =<entry>= is pretty ugly. Really what I want is
}
#+end_src
**** Strip Links from Strings
*** Strip Links from Strings
#+begin_src rust :tangle src/export_atom.rs
fn strip_links_from_str(in_str: &str) -> Result<String> {
@ -1494,7 +1502,7 @@ fn strip_links_from_str(in_str: &str) -> Result<String> {
}
#+end_src
**** Convert my org-style timestamps to RFC-3339 strings
*** Convert my org-style timestamps to RFC-3339 strings
#+begin_src rust :tangle src/export_atom.rs
fn rfcize_datestamp(in_str: String) -> Result<String> {
@ -1517,7 +1525,7 @@ fn rfcize_datestamp(in_str: String) -> Result<String> {
}
#+end_src
**** Internal Buffer Writer for Escaping Entities
*** Internal Buffer Writer for Escaping Entities
I implemented a really simple/dumb =Write= interface so will =HtmlEscape= things which are written to it. I should make this take a String under the hood instead of =Vec<bytes>= but meh it's good enough for now.
@ -1578,7 +1586,7 @@ impl Write for &mut InternalWriter {
}
#+end_src
*** Second API
** Second API
there's a step further on, where an API takes a list of headings and feed metadata, and it parses each heading and its subheadings to HTML, *which is an API I already want to provide to document systems*. it could take arbitrary document headings provided through the public interface, and construct multi-page feeds.
@ -1586,11 +1594,11 @@ this requires the ability to export only a given subheading, which I could imple
or we could just clobber together a version of [[https://github.com/tanrax/RSSingle][RSSingle]]; [[id:personal_software_can_be_shitty][Personal Software Can Be Shitty]].
*** Future API
** Future API
way out there: how do feed readers behave if the "feed" is just the linearized document with updated-at and whatnot applied to it? The feed would send the entire page with each update, but what if each heading could then be processed in to a diff or summary of changes? how could i possibly do that well, anyhow?
** Library definition and exports for the Python library
* Library definition and exports for the native Python library
:PROPERTIES:
:header-args:rust: :tangle src/lib.rs :mkdirp yes
:END:
@ -1634,12 +1642,12 @@ fn arroyo_rs(py: Python, m: &PyModule) -> PyResult<()> {
}
#+end_src
*** NEXT it would be cool if the =htmlize_file= call could take =**kwargs= and construct the =ExportOptions= itself.
** NEXT it would be cool if the =htmlize_file= call could take =**kwargs= and construct the =ExportOptions= itself.
This makes it easy to make the same interface for =atomize_file=
*** WAITING add =atomize_file= to the =pyfn='s
** Tests
** WAITING add =atomize_file= to the =pyfn='s
* Code Unit Tests
I wrote Cargo tests for [[id:20231023T130916.139809][split_quoted_string]], and some simple parser tests.
@ -1857,88 +1865,11 @@ if __name__ == "__main__":
cli()
#+end_src
** Persisting a single file to the Database
:PROPERTIES:
:ID: 20231026T180934.959369
:ROAM_ALIASES: arroyo.persist_one_file
:END:
** NEXT Usage
*This uses the SQLModel interface but I'm leaving it here without tangles cause it's useful.*
See [[id:20231217T154857.983742][Arcology ingestfiles Command]] for examples of the parser interface
,:header-args:python: :tangle arroyo/__init__.py :mkdirp yes
=arroyo.persist_one_file= takes a path and returns either a sqlite persisted Document or None. It's a bit complicated, so let's take a look at this step by step:
- Check if the doc's hash does not match the hash of a prior run
#+begin_src python
import sys
from sqlmodel import Session
from . import models
from .arroyo_rs import parse_file, InvalidDocError
def persist_one_file(s: Session, path: str) -> models.Document | None:
existing_doc = s.get(models.Document, path)
if existing_doc and not models.document.hash_updated(existing_doc):
return None
if existing_doc:
s.delete(existing_doc)
#+end_src
- Try to parse the doc
#+begin_src python
try:
doc = parse_file(path)
except InvalidDocError:
sys.stderr.write(f"Skipping invalid doc {path}\n")
return None
#+end_src
- Collect the keywords to see if it's an Arcology-published document
#+begin_src python
kws = map(lambda kw: kw.keyword, doc.keywords)
kws = filter(lambda kw: kw.upper() == "ARCOLOGY_KEY", kws)
create_documents = len(list(kws)) > 0
#+end_src
- Unconditionally import the Keywords in to the database so that other Arroyo generators don't rely on pages being published.
#+begin_src python
keywords = models.Keyword.from_native_doc(doc)
[s.merge(keyword) for keyword in keywords]
#+end_src
- If the page shall be published, it will persist the document and metadata required to build an Arcology page.
#+begin_src python
if create_documents:
tags = models.Tag.from_native_doc(doc)
references = models.Reference.from_native_doc(doc)
headings = models.Heading.from_native_doc(doc)
document = models.Document.from_native_doc(doc)
s.add(document)
[s.merge(tag) for tag in tags]
[s.merge(reference) for reference in references]
[s.merge(heading) for heading in headings]
s.commit()
s.refresh(document)
sys.stderr.write(f"Persisted document for {path}\n")
return document
#+end_src
- Otherwise it'll just commit the keywords merged in above.
#+begin_src python
else:
sys.stderr.write(f"Only keywords for doc {path}\n")
s.commit() # can't batch because of the upsert of tags...
#+end_src
See [[id:20240202T144002.656093][Arcology's =org_page= HTTP Handler]] and the atom endpoint for examples of the exporter interface
* Addendum: Making sure =org-auto-tangle= works with this buffer and MELPA languages
:PROPERTIES:
@ -1963,3 +1894,12 @@ IDK why but org-auto-tangle is struggling with files that include types from ext
(require 'rust-mode)
(require 'nix-mode))))
#+end_src
* Future Work
** NEXT move [[id:arcology/django/roam][Arcology Roam Models]]
** NEXT move [[id:arroyo/django/generators][The Arroyo Generators]]
** NEXT remove the old elisp logic
** NEXT move [[id:arcology/django/interfaces][Arcology CLI Interface]] [[id:20231217T154938.132553][Arcology generate Command]]
** NEXT [#C] source code block extraction & babel execution?
how wild would it be to have a little wasm environment in the rust to execute code blocks? at the very least i want to build a tangler and some Python modules to populate Arroyo code generators.

81
arroyo-nixos.el
View File

@ -1,79 +1,9 @@
(defcustom arroyo-nixos-network-path (expand-file-name "~/arroyo-nix/networks/laptops.nix" org-roam-directory)
"Location of Nix network definition suitable for Morphing"
:group 'arroyo
:type 'string)
(dolist (mod '("ARROYO_NIXOS_MODULE"
"ARROYO_SYSTEM_ROLE"
"ARROYO_SYSTEM_EXCLUDE"))
(add-to-list 'arroyo-db-keywords mod nil #'equal))
(add-to-list 'arroyo-db--schemata
'(nixos
[(file :not-null)
(nix-module :not-null)
role]))
(defun arroyo-nixos-update-db (&optional keywords-too)
(interactive "P")
(when keywords-too
(arroyo-db-update-all-roam-files))
(if (org-roam-file-p)
(arroyo-db-query [:delete :from nixos :where (= file $s1)]
(buffer-file-name))
(arroyo-db-query [:delete :from nixos]))
(when-let* ((args
(if (org-roam-file-p)
(list [:select [file value] :from keywords
:where (= keyword $s1) :and (= file $s2)]
"ARROYO_NIXOS_MODULE"
(buffer-file-name))
(list [:select [file value] :from keywords
:where (= keyword $s1)]
"ARROYO_NIXOS_MODULE")))
(results (apply #'arroyo-db-query args))
(results (-map (pcase-lambda (`(,file ,value))
(list file value (arroyo-db-get "ARROYO_SYSTEM_ROLE" file)))
results)))
(arroyo-db-query [:insert :into nixos :values $v1]
(-map (lambda (result) (apply #'vector result))
results))))
(add-to-list 'arroyo-db-update-functions #'arroyo-nixos-update-db)
;; [[file:arroyo-nixos.org::*Calling in to the Arcology to generate \[\[id:cce/my_nixos_configuration\]\[My NixOS Configuration\]\]][Calling in to the Arcology to generate [[id:cce/my_nixos_configuration][My NixOS Configuration]]:1]]
(defun arroyo-nixos-imports (&optional role)
(if-let* ((imports (arroyo-nixos-imports1))
(no-role? (not role)))
(-map #'car imports)
(progn
(let* ((all-files (-map #'cadr imports))
(role-only-files (arroyo-db-by-keyword "ARROYO_SYSTEM_ROLE" role))
(exclude-these-files (arroyo-db-by-keyword "ARROYO_SYSTEM_EXCLUDE" role))
;; also need to exclude things whose ARROYO_SYSTEM_ROLE is not `role'!
(not-my-role-files
(-map #'car
(arroyo-db-query [:select file :from nixos
:where (!= role $s1)]
(list role))))
;; combine all those lists in to a single authoritative list.
(amalgamated-files
(-union role-only-files
(->> all-files
(--remove (-contains? exclude-these-files it))
(--remove (-contains? not-my-role-files it)))))
;; make them relative to org-roam-directory for the module orderer
(rel-files (--map (file-relative-name it org-roam-directory) amalgamated-files))
;; map the org path back to the tangled module
(tangled-files (-map (lambda (org-file)
(car (arroyo-db-get "ARROYO_NIXOS_MODULE"
(expand-file-name org-file org-roam-directory))))
(arroyo-utils-order-modules rel-files))))
(-filter #'identity tangled-files)))))
(defun arroyo-nixos-imports1 ()
(->> [:select [nix-module file] :from nixos]
(arroyo-db-query)))
(arroyo-generate-imports "nixos" role))
;; Calling in to the Arcology to generate [[id:cce/my_nixos_configuration][My NixOS Configuration]]:1 ends here
;; [[file:arroyo-nixos.org::*Flooding the Arroyo tangles and deploys all sub-systems][Flooding the Arroyo tangles and deploys all sub-systems:1]]
(defun arroyo-nixos-full-spread (host &optional local-only switch-action)
"Flood the arroyo. Tangle all related files and run morph deploy.
DEP-FILE is a string pointing to a Morph network definition.
@ -91,5 +21,8 @@ SWITCH-ACTION is one of the actions which morph deploy expects. Read its docs."
"*nix-deploy*")))
(defalias 'arroyo-flood 'arroyo-nixos-full-spread)
;; Flooding the Arroyo tangles and deploys all sub-systems:1 ends here
;; [[file:arroyo-nixos.org::*Flooding the Arroyo tangles and deploys all sub-systems][Flooding the Arroyo tangles and deploys all sub-systems:2]]
(provide 'arroyo-nixos)
;; Flooding the Arroyo tangles and deploys all sub-systems:2 ends here

100
arroyo-nixos.org
View File

@ -2,6 +2,7 @@
:ID: arroyo/nixos
:END:
#+TITLE: Arroyo NixOS Generator
#+filetags: :Project:
#+PROPERTY: header-args :mkdirp yes
#+PROPERTY: header-args:emacs-lisp :tangle arroyo-nixos.el
#+AUTO_TANGLE: t
@ -12,104 +13,18 @@ Arroyo can manage any number of NixOS hosts, though right now it seeks to only m
Server and cluster management will be explored at some point in the future, but for now this is designed to build laptops and desktops which can be given a bare-bones NixOS install through my [[id:cce/nixos_automatic_partitioning_installer][NixOS Automatic Partitioning Installer]] and attached to the same network as a machine running Arroyo or capable of running [[id:cce/morph][Morph]] on a set of Arroyo tangled files.
* NixOS =configuration.nix= Generator
* Calling in to the Arcology to generate [[id:cce/my_nixos_configuration][My NixOS Configuration]]
This thing is pretty functionally similar to the [[id:arroyo/home-manager][Arroyo Home Manager]] generator and is conceptually quite similar. It constructs a [[id:c75d20e6-8888-4c5a-ac97-5997e2f1c711][NixOS]] =configuration.nix=-like which is suitable for being deployed with [[id:cce/morph][Morph]]. That file documents [[id:cce/morph-laptops][My Laptops]] and provides an example of the system "in-flight."
I'll move this code here someday, for now we call it with a nix run. good luck making this work for yourself! you could replace the =path:= and it might just work.
#+begin_src emacs-lisp
(defcustom arroyo-nixos-network-path (expand-file-name "~/arroyo-nix/networks/laptops.nix" org-roam-directory)
"Location of Nix network definition suitable for Morphing"
:group 'arroyo
:type 'string)
(dolist (mod '("ARROYO_NIXOS_MODULE"
"ARROYO_SYSTEM_ROLE"
"ARROYO_SYSTEM_EXCLUDE"))
(add-to-list 'arroyo-db-keywords mod nil #'equal))
(add-to-list 'arroyo-db--schemata
'(nixos
[(file :not-null)
(nix-module :not-null)
role]))
#+end_src
#+begin_src emacs-lisp
(defun arroyo-nixos-update-db (&optional keywords-too)
(interactive "P")
(when keywords-too
(arroyo-db-update-all-roam-files))
(if (org-roam-file-p)
(arroyo-db-query [:delete :from nixos :where (= file $s1)]
(buffer-file-name))
(arroyo-db-query [:delete :from nixos]))
(when-let* ((args
(if (org-roam-file-p)
(list [:select [file value] :from keywords
:where (= keyword $s1) :and (= file $s2)]
"ARROYO_NIXOS_MODULE"
(buffer-file-name))
(list [:select [file value] :from keywords
:where (= keyword $s1)]
"ARROYO_NIXOS_MODULE")))
(results (apply #'arroyo-db-query args))
(results (-map (pcase-lambda (`(,file ,value))
(list file value (arroyo-db-get "ARROYO_SYSTEM_ROLE" file)))
results)))
(arroyo-db-query [:insert :into nixos :values $v1]
(-map (lambda (result) (apply #'vector result))
results))))
(add-to-list 'arroyo-db-update-functions #'arroyo-nixos-update-db)
#+end_src
=arroyo-home-manager-imports= returns a list of strings suitable for =import='ing in [[id:f32b616e-ede6-461e-8bae-b5f1c717a05e][home-manager's literate helpers]].
#+NAME: arroyo-nixos-imports
#+begin_src emacs-lisp
(defun arroyo-nixos-imports (&optional role)
(if-let* ((imports (arroyo-nixos-imports1))
(no-role? (not role)))
(-map #'car imports)
(progn
(let* ((all-files (-map #'cadr imports))
(role-only-files (arroyo-db-by-keyword "ARROYO_SYSTEM_ROLE" role))
(exclude-these-files (arroyo-db-by-keyword "ARROYO_SYSTEM_EXCLUDE" role))
;; also need to exclude things whose ARROYO_ARROYO_SYSTEM_EXCLUDErole'!
(not-my-role-files
(-map #'car
(arroyo-db-query [:select file :from nixos
:where (!= role $s1)]
(list role))))
;; combine all those lists in to a single authoritative list.
(amalgamated-files
(-union role-only-files
(->> all-files
(--remove (-contains? exclude-these-files it))
(--remove (-contains? not-my-role-files it)))))
;; make them relative to org-roam-directory for the module orderer
(rel-files (--map (file-relative-name it org-roam-directory) amalgamated-files))
;; map the org path back to the tangled module
(tangled-files (-map (lambda (org-file)
(car (arroyo-db-get "ARROYO_NIXOS_MODULE"
(expand-file-name org-file org-roam-directory))))
(arroyo-utils-order-modules rel-files))))
(-filter #'identity tangled-files)))))
(defun arroyo-nixos-imports1 ()
(->> [:select [nix-module file] :from nixos]
(arroyo-db-query)))
(arroyo-generate-imports "nixos" role))
#+end_src
** NEXT I should make this function which crates the tangled-files list a function in [[id:arroyo/system-cache][Arroyo System Cache]]
It's a very useful thing to say "just go figure out the DAG stuff for the files that are set up by -these- metadata keywords" -- i already have a copy of this in [[id:arroyo/home-manager][Arroyo Home Manager]] that could be combined in to a better pattern, surely.
I need to generally be thinking about an "API" for arroyo, something that other folks could extend, rather than just accreting features forever.
* Arroyo NixOS manages systems with [[id:cce/morph][Morph]]:
It goes in to [[id:arroyo/home-manager][Arroyo Home Manager]] but can be installed from GitHub to bootstrap the system.
It goes in to [[id:arroyo/home-manager][Arroyo Home Manager]] but can be installed from GitHub to bootstrap the system if necessary.
#+ARROYO_HOME_MODULE: hm/morph.nix
@ -120,20 +35,21 @@ It goes in to [[id:arroyo/home-manager][Arroyo Home Manager]] but can be install
}
#+end_src
** NEXT why isn't this on the [[id:cce/morph][Morph]] page..?
* Flooding the Arroyo tangles and deploys all sub-systems
:PROPERTIES:
:ID: 20211130T215142.470274
:ROAM_ALIASES: arroyo-flood
:END:
an interactive command which stitches together the arroyo system and runs the =morph= command which deploys the system:
=arroyo-flood= is an interactive command which stitches together the arroyo system and runs the =morph= command which deploys the system:
#+ARROYO_MODULE_WANTS: cce/org-roam.org
[[elisp:(arroyo-flood)][Click here to flood the Arroyo.]]
#+begin_src emacs-lisp :results none
(defun arroyo-nixos-full-spread (host &optional local-only switch-action)
"Flood the arroyo. Tangle all related files and run morph deploy.
DEP-FILE is a string pointing to a Morph network definition.

51
arroyo.org
View File

@ -14,15 +14,13 @@
:ID: arroyo/what-are-arroyos
:END:
Arroyo systems are the latest iteration of my the [[id:cce/cce][CCE]] Emacs =init.el= generator cum [[id:60f710b2-6a1f-44be-bc13-dfe01e46d4e3][Concept Operating System]] built for managing my laptops and desktops around an [[id:c75d20e6-8888-4c5a-ac97-5997e2f1c711][NixOS]], [[id:cce/emacs][Emacs]], [[id:cce/exwm][EXWM]], and [[id:cce/evil_mode][Evil Mode]] system. It is distributed and developed as a collection of [[id:1fb8fb45-fac5-4449-a347-d55118bb377e][org-mode]] documents which can be used to provision and upgrade the system and shared on the web to enrich the lives of weirdos like [[id:ryan_rix][me]].
Arroyo systems are the underlying design systems for the [[id:cce/cce][CCE]] Emacs =init.el= generator and [[id:60f710b2-6a1f-44be-bc13-dfe01e46d4e3][Concept Operating System]]. It's used for managing my laptops' and desktop's and servers' and mobiles' software around an [[id:c75d20e6-8888-4c5a-ac97-5997e2f1c711][NixOS]], [[id:cce/emacs][Emacs]], [[id:cce/kde_is_a_base_for_my_emacs_desktop][KDE]], and [[id:cce/evil_mode][Evil Mode]]. Arroyo Systems are distributed and developed as a collection of [[id:1fb8fb45-fac5-4449-a347-d55118bb377e][org-mode]] documents which can be used to provision and upgrade the system and are published on the web to enrich the lives of weirdos like [[id:ryan_rix][me]].
pulling inspiration from [[id:5dceb633-823f-41a5-b39b-ad37b1b65e0a][vulpea]] and [[id:43523a67-a951-4464-9af4-8a8fe116281a][vino]] of building out a context-specific database [[id:128ab0e8-a1c7-48bf-9efe-0c23ce906a48][hypermedia application]] which integrate with [[id:cce/org-roam][org-roam]] rather than extending or working within the API-space of [[id:cce/org-roam][org-roam]] itself, Arroyo consists of a database caching layer working alongside an [[id:cce/org-roam][org-roam]] database which can be queried to assemble a [[id:c75d20e6-8888-4c5a-ac97-5997e2f1c711][NixOS]] system and highly-customized [[id:cce/emacs][Emacs]] installation-and-configuration.
Arroyo allows you to build context-specific databases from a sort of [[id:128ab0e8-a1c7-48bf-9efe-0c23ce906a48][hypermedia application]] which integrate with [[id:cce/org-roam][org-roam]]. Rather than extending or working within the API-space of [[id:cce/org-roam][org-roam]] itself, Arroyo is an org-mode parser and metadata extractor, and a database cache model layer which can be queried to assemble a [[id:c75d20e6-8888-4c5a-ac97-5997e2f1c711][NixOS]] system and highly-customized [[id:cce/emacs][Emacs]] installation-and-configuration.
it's designed such that a user can plug various pieces of system configuration together in the form of [[id:1fb8fb45-fac5-4449-a347-d55118bb377e][org-mode]] documents, creating a system which is literally self-documented.
it's designed such that a user can plug various pieces of system configuration together in the form of [[id:1fb8fb45-fac5-4449-a347-d55118bb377e][org-mode]] documents downloaded or synced from trustworthy friends, creating a system which is literally ([[id:cce/literate_programming][literately]]) self-documented.
By using a cache similar to the one built for my [[id:e050ed03-044f-4a48-912d-72579bef9a26][org-roam keyword caching]] patch, we can construct a higher-level database schema for the Arroyo system generators. The [[id:arroyo/system-cache][Arroyo System Cache]] aka =arroyo-db= will create a simple key-value store which higher-level generators like [[id:arroyo/emacs][Arroyo Emacs Generator]] can use to assemble and automate systems.
An Arroyo System's configuration can be published on the web using [[id:1d917282-ecf4-4d4c-ba49-628cbb4bb8cc][Arcology]], an elixir web document publishing framework built under similar ideas and philosophies with the end goal of socially-maintained hand-crafted and user-customized Linux systems.
An Arroyo System's configuration can be published on the web using [[id:1d917282-ecf4-4d4c-ba49-628cbb4bb8cc][Arcology]], an python web document publishing framework built using Arroyo's parser and philosophies with the end goal of managing socially-maintained, hand-crafted, and user-customized Linux systems.
* Arroyo Systems are "designed" to develop in fits and bursts
:PROPERTIES:
@ -38,14 +36,17 @@ The Arroyo system pulls inspiration from this as a lens to look at how personal
- external changes and new tools should be easy to try and easy to then disregard
- slow thinking and quick implementation of internal development to match with life needs
- management of overgrowth through periods of careful usage and evaluation
- an oasis path that cuts through the status quo
This system develops when I feel like it, and has slowly over the course of a decade developed in to something which can be shared and perhaps replicated. When a beautiful summer's day spurs me to investigate a new library or I learn about a new paradigm I sit and think about it. The Arroyo floods and growth follows quickly; there might not be another rain of insight until winter, after all.
* NEXT Assembling an Arroyo System
[[id:20231113T195508.942155][Rebuild of The Complete Computer]]
- How do I use these libraries to generate an [[id:cce/emacs][Emacs]] init file?
- How do I use these libraries to build a NixOS system?
- How do I use these libraries to build an emacs-gcc with packages provided by NixOS at build time?
- How do I use these libraries to build an emacs-native-comp with packages provided by NixOS at build time?
- How do I use these libraries to install and manage software for my user account?
- How do I build more generators on top of this system?
@ -59,15 +60,26 @@ It should probably be defined as a [[id:arroyo/home-manager][Arroyo Home Manager
#+begin_src emacs-lisp :tangle ../cce/arroyo.el :comments link :results none
(add-to-list 'load-path "~/org/arroyo")
(require 'org-roam)
(require 'arroyo-db)
(require 'arroyo-utils)
; (require 'arroyo-db)
; (require 'arroyo-utils)
(require 'arroyo-home-manager)
(require 'arroyo-nixos)
(require 'arroyo-emacs)
(require 'arroyo-feeds)
; (require 'arroyo-feeds)
#+end_src
* Arroyo Generators
* Structure of the Arroyo (Or: Table of Contents)
(These are spread across multiple projects right now, I am still nailing down the exact structure of the Arroyo, Arcology, and CCE)
- [[id:20231023T115950.248543][The arroyo_rs Native Org Parser]]
- [[id:20231023T115950.248543][Arroyo Org Parser]]
- [[id:20240112T120813.386800][Arroyo HTML exporter]]
- [[id:arcology/django/roam][Arcology Roam Models]]
- [[id:arroyo/django/generators][The Arroyo Generators]]
- [[id:arcology/django/interfaces][Arcology CLI Interface]]
* Arroyo Generators :noexport:
Working arroyos:
- [[file:arroyo-nixos.org][Arroyo Nixos]] generated NixOS system, which includes
@ -84,3 +96,20 @@ it's not really built to compose, and i'd rather not use the arroyo system as a
Coming back to this [2021-12-02 Thu] -- [[id:20211202T201728.221203][i specify a table layout]] which could be expressed as an Arroyo generator. I'd like to keep the generator outside of the Arroyo project, and in Arcology, but I think at this point it makes sense to spend a bit of time cleaning up the core APIs and documenting them better, and then coming back to Arcology with a smarter table design.
** NEXT arroyo module groupings to modularize [[id:cce/cce][CCE]]
* Greetz and Thanks
the emacs and nixpkgs communities
[[id:cce/org-roam][org-roam]]
pulling inspiration from [[id:5dceb633-823f-41a5-b39b-ad37b1b65e0a][vulpea]] and [[id:43523a67-a951-4464-9af4-8a8fe116281a][vino]]
[[id:f4d0be16-1f68-4598-a02c-0327759e034c][Torbjorn Loken]]
alphapapa
[[id:7f451675-1db0-4093-9d8e-28cc5d597545][maya kate]]
Yoshiki Shmitz (RIP to a real one)
[[id:5e25597e-ae0f-4f15-ae27-f89351e2ade3][Malleable Systems Collective]]

31
arroyo_rs.org
View File

@ -1,31 +0,0 @@
:PROPERTIES:
:ID: 20231021T165413.872542
:ROAM_ALIASES: "Arroyo Org Parser" arroyo-rs
:ROAM_REFS: https://code.rix.si/rrix/arroyo_rs
:END:
#+TITLE: arroyo-rs is an org-mode document parser for Python and Rust
#+FILETAGS: :Project:
=arroyo-rs= is an [[https://orgmode.org/][org-mode]] parser library for [[https://engine.arcology.garden][The Arcology Project]]. It exposes a simple interface to the [[https://www.rust-lang.org/][Rust]] [[https://docs.rs/orgize/][Orgize]] library, wrapped in [[https://pyo3.rs/][pyo3]] bindings to be accessible from [[https://python.org][Python]]. Most of the Org-mode parsers I've looked at so far lack certain features or parsing capabilities that would make it difficult or impossible to implement [[https://cce.whatthefuck.computer/arroyo/][Arroyo]] and the Arcology Project.
This package exports two public functions and a handful of types:
- =arroyo_rs.parse_file(path: str) -> Document=
- =arroyo_rs.export_html(path: str) -> str= (This hasn't been implemented yet and the shape of it might change, for example adding DB-backed link rewriting for the Arcology)
- and types which map to the Arcology Project's database types.
* Further Work :Tasks:
** NEXT pull =arroyo_rs= in to literate docs
** NEXT evaluate moving [[id:arcology/arroyo-page][Arroyo Arcology Generator]] in to this repo
- SQLModel python stuff for DB persistence
** NEXT move keyword and file table generation to arroyo_rs
** NEXT move the emacs, nixos, and home-manager generators across
** NEXT source code block extraction & babel execution?
how wild would it be to have a little wasm environment in the rust to execute code blocks? at the very least i want to build a tangler and some Python modules to populate Arroyo code generators.