switch arroyo-emacs to shell out to Arcology Generators

re-jiggle link handling to mark missing ID links as 404
make sure htmlize_file and its options are in the public iface
2023-12-27 19:32:08 -08:00 · 2023-12-27 19:31:41 -08:00 · 2023-12-27 18:19:33 -08:00 · 2023-12-27 18:19:12 -08:00 · 2023-12-27 18:18:48 -08:00 · 2023-12-18 23:05:06 -08:00
28 changed files with 3555 additions and 135 deletions
--- a/.envrc
+++ b/.envrc
@ -0,0 +1 @@
+use flake
--- a/.gitignore
+++ b/.gitignore
@ -0,0 +1,9 @@
+/target
+/.direnv
+/result
+/venv/
+
+__pycache__
+
+/arroyo.db
+arroyo_rs.*.so
--- a/Cargo.lock
+++ b/Cargo.lock
@ -0,0 +1,794 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "adler"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+
+[[package]]
+name = "aho-corasick"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "anyhow"
+version = "1.0.75"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
+
+[[package]]
+name = "arroyo-rs"
+version = "0.0.1"
+dependencies = [
+ "anyhow",
+ "itertools",
+ "lexpr",
+ "orgize",
+ "pyo3",
+ "regex",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+
+[[package]]
+name = "base64"
+version = "0.21.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9"
+
+[[package]]
+name = "bincode"
+version = "1.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
+[[package]]
+name = "bytecount"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1a12477b7237a01c11a80a51278165f9ba0edd28fa6db00a65ab230320dc58c"
+
+[[package]]
+name = "cc"
+version = "1.0.83"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "crc32fast"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "deranged"
+version = "0.3.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc"
+dependencies = [
+ "powerfmt",
+]
+
+[[package]]
+name = "either"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
+
+[[package]]
+name = "equivalent"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
+
+[[package]]
+name = "flate2"
+version = "1.0.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide",
+]
+
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
+[[package]]
+name = "hashbrown"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+
+[[package]]
+name = "hashbrown"
+version = "0.14.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
+
+[[package]]
+name = "heck"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+
+[[package]]
+name = "indexmap"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
+dependencies = [
+ "autocfg",
+ "hashbrown 0.12.3",
+ "serde",
+]
+
+[[package]]
+name = "indexmap"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f"
+dependencies = [
+ "equivalent",
+ "hashbrown 0.14.3",
+]
+
+[[package]]
+name = "indextree"
+version = "4.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c40411d0e5c63ef1323c3d09ce5ec6d84d71531e18daed0743fccea279d7deb6"
+
+[[package]]
+name = "indoc"
+version = "2.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8"
+
+[[package]]
+name = "itertools"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
+
+[[package]]
+name = "jetscii"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47f142fe24a9c9944451e8349de0a56af5f3e7226dc46f3ed4d4ecc0b85af75e"
+
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+
+[[package]]
+name = "lazycell"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
+
+[[package]]
+name = "lexpr"
+version = "0.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a84de6a9df442363b08f5dbf0cd5b92edc70097b89c4ce4bfea4679fe48bc67"
+dependencies = [
+ "itoa",
+ "lexpr-macros",
+ "ryu",
+]
+
+[[package]]
+name = "lexpr-macros"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36b5cb8bb985c81a8ac1a0f8b5c4865214f574ddd64397ef7a99c236e21f35bb"
+dependencies = [
+ "proc-macro2",
+ "quote",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.149"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b"
+
+[[package]]
+name = "line-wrap"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f30344350a2a51da54c1d53be93fade8a237e545dbcc4bdbe635413f2117cab9"
+dependencies = [
+ "safemem",
+]
+
+[[package]]
+name = "linked-hash-map"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
+
+[[package]]
+name = "lock_api"
+version = "0.4.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45"
+dependencies = [
+ "autocfg",
+ "scopeguard",
+]
+
+[[package]]
+name = "memchr"
+version = "2.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167"
+
+[[package]]
+name = "memoffset"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
+[[package]]
+name = "miniz_oxide"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"
+dependencies = [
+ "adler",
+]
+
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
+
+[[package]]
+name = "onig"
+version = "6.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f"
+dependencies = [
+ "bitflags",
+ "libc",
+ "once_cell",
+ "onig_sys",
+]
+
+[[package]]
+name = "onig_sys"
+version = "69.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7"
+dependencies = [
+ "cc",
+ "pkg-config",
+]
+
+[[package]]
+name = "orgize"
+version = "0.9.0"
+source = "git+https://code.rix.si/rrix/orgize#47307192a7b01fa92f6cbc973544e392515c69ba"
+dependencies = [
+ "bytecount",
+ "indexmap 1.9.3",
+ "indextree",
+ "jetscii",
+ "lazy_static",
+ "memchr",
+ "nom",
+ "serde",
+ "serde_indextree",
+ "syntect",
+]
+
+[[package]]
+name = "parking_lot"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
+dependencies = [
+ "lock_api",
+ "parking_lot_core",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.9.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall",
+ "smallvec",
+ "windows-targets",
+]
+
+[[package]]
+name = "pkg-config"
+version = "0.3.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
+
+[[package]]
+name = "plist"
+version = "1.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5699cc8a63d1aa2b1ee8e12b9ad70ac790d65788cd36101fa37f87ea46c4cef"
+dependencies = [
+ "base64",
+ "indexmap 2.1.0",
+ "line-wrap",
+ "quick-xml",
+ "serde",
+ "time",
+]
+
+[[package]]
+name = "powerfmt"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "pyo3"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04e8453b658fe480c3e70c8ed4e3d3ec33eb74988bd186561b0cc66b85c3bc4b"
+dependencies = [
+ "anyhow",
+ "cfg-if",
+ "indoc",
+ "libc",
+ "memoffset",
+ "parking_lot",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+ "unindent",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a96fe70b176a89cff78f2fa7b3c930081e163d5379b4dcdf993e3ae29ca662e5"
+dependencies = [
+ "once_cell",
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "214929900fd25e6604661ed9cf349727c8920d47deff196c4e28165a6ef2a96b"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dac53072f717aa1bfa4db832b39de8c875b7c7af4f4a6fe93cdbf9264cf8383b"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7774b5a8282bd4f25f803b1f0d945120be959a36c72e08e7cd031c792fdfd424"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "quick-xml"
+version = "0.31.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa"
+dependencies = [
+ "bitflags",
+]
+
+[[package]]
+name = "regex"
+version = "1.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax 0.8.2",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax 0.8.2",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.6.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
+
+[[package]]
+name = "ryu"
+version = "1.0.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"
+
+[[package]]
+name = "safemem"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072"
+
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
+[[package]]
+name = "scopeguard"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+
+[[package]]
+name = "serde"
+version = "1.0.189"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.189"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_indextree"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a70e7d58005454b0ba863a8775df0c4391a8128c3a34974001de556d661bf74f"
+dependencies = [
+ "indextree",
+ "serde",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.108"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "smallvec"
+version = "1.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a"
+
+[[package]]
+name = "syn"
+version = "2.0.38"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syntect"
+version = "4.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b20815bbe80ee0be06e6957450a841185fcf690fe0178f14d77a05ce2caa031"
+dependencies = [
+ "bincode",
+ "bitflags",
+ "flate2",
+ "fnv",
+ "lazy_static",
+ "lazycell",
+ "onig",
+ "plist",
+ "regex-syntax 0.6.29",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "walkdir",
+ "yaml-rust",
+]
+
+[[package]]
+name = "target-lexicon"
+version = "0.12.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14c39fd04924ca3a864207c66fc2cd7d22d7c016007f9ce846cbb9326331930a"
+
+[[package]]
+name = "time"
+version = "0.3.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5"
+dependencies = [
+ "deranged",
+ "itoa",
+ "powerfmt",
+ "serde",
+ "time-core",
+ "time-macros",
+]
+
+[[package]]
+name = "time-core"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
+
+[[package]]
+name = "time-macros"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20"
+dependencies = [
+ "time-core",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
+
+[[package]]
+name = "unindent"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce"
+
+[[package]]
+name = "walkdir"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee"
+dependencies = [
+ "same-file",
+ "winapi-util",
+]
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-util"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
+[[package]]
+name = "windows-targets"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
+
+[[package]]
+name = "yaml-rust"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
+dependencies = [
+ "linked-hash-map",
+]
--- a/Cargo.toml
+++ b/Cargo.toml
@ -0,0 +1,24 @@
+# [[file:arroyo-native-parser.org::*Rust package definition][Rust package definition:1]]
+[package]
+name = "arroyo-rs"
+version = "0.0.1"
+edition = "2021"
+# Rust package definition:1 ends here
+
+# [[file:arroyo-native-parser.org::*Rust package definition][Rust package definition:2]]
+[lib]
+name = "arroyo_rs"
+crate-type = ["cdylib", "rlib"]
+# Rust package definition:2 ends here
+
+# [[file:arroyo-native-parser.org::*Rust package definition][Rust package definition:3]]
+[dependencies]
+lexpr = "0.2.7"
+# orgize = "0.9.0"
+orgize.git = "https://code.rix.si/rrix/orgize"
+orgize.features = ["syntect"]
+anyhow = "1.0.75"
+pyo3 = { version = "0.20.0", features = ["anyhow"] }
+itertools = "0.11.0"
+regex = "1.10.2"
+# Rust package definition:3 ends here
--- a/arroyo-db.el
+++ b/arroyo-db.el
@ -153,24 +153,24 @@ SQL can be either the emacsql vector representation, or a string."

 (defun arroyo--files-for-role (role)
  (->> role
-       (arroyo-db-by-keyword "ARROYO_NIXOS_ROLE") 
+       (arroyo-db-by-keyword "ARROYO_SYSTEM_ROLE")
       (--filter (arroyo-db-get "ARROYO_NIXOS_MODULE" it))
       (--map (cons it (arroyo-db--get-file-title-from-org-roam it)))))

 (defun arroyo-db--get-file-title-from-org-roam (file)
  (->> file
-       (org-roam-db-query [:select title :from nodes :where (= level 0) :and (= file $s1)]) 
+       (org-roam-db-query [:select title :from nodes :where (= level 0) :and (= file $s1)])
       (caar)))
 ;; headmatter & db functions:5 ends here

-;; [[file:arroyo-system-cache.org::*Arroyo DB uses hashes, not incremental versioning][Arroyo DB uses hashes, not incremental versioning:1]]
+;; [[file:arroyo-system-cache.org::*CANCELLED Arroyo DB uses hashes, not incremental versioning][CANCELLED Arroyo DB uses hashes, not incremental versioning:1]]
 (defvar arroyo-db-hash-input-functions
  '(arroyo-db--hash)
  "A list of functions which take an accumulator and hash their
  own configuraiton in to the accumulator")
-;; Arroyo DB uses hashes, not incremental versioning:1 ends here
+;; CANCELLED Arroyo DB uses hashes, not incremental versioning:1 ends here

-;; [[file:arroyo-system-cache.org::*Arroyo DB uses hashes, not incremental versioning][Arroyo DB uses hashes, not incremental versioning:2]]
+;; [[file:arroyo-system-cache.org::*CANCELLED Arroyo DB uses hashes, not incremental versioning][CANCELLED Arroyo DB uses hashes, not incremental versioning:2]]
 (defun arroyo-db--version ()
  (-reduce (lambda (first second)
             (funcall second first))
@ -180,7 +180,7 @@ SQL can be either the emacsql vector representation, or a string."
 (defun arroyo-db--hash (acc)
  "A function which mixes the databases schemata in to the arroyo version hash, used to invalidate the DB on change"
  (secure-hash 'sha256 (format "%s" (append acc arroyo-db-keywords))))
-;; Arroyo DB uses hashes, not incremental versioning:2 ends here
+;; CANCELLED Arroyo DB uses hashes, not incremental versioning:2 ends here

 ;; [[file:arroyo-system-cache.org::*org-roam #+PROPERTY keyword caching][org-roam #+PROPERTY keyword caching:1]]
 (defcustom arroyo-db-keywords '("BIRTHDAY" "PRONOUNS" "LOCATION")
@ -284,7 +284,7 @@ Return the number of rows inserted."
    (org-with-point-at 1
      (dolist (prop props ret)
        (when-let ((v (org-entry-get (point) prop)))
-          (push (cons prop v) ret)))))) 
+          (push (cons prop v) ret))))))

 (defun arroyo-db--extract-global-props (props)
  "Extract PROPS from the current Org buffer.
--- a/arroyo-emacs.el
+++ b/arroyo-emacs.el
@ -53,18 +53,26 @@

 ;; [[file:arroyo-emacs.org::*Arroyo Emacs =init.el= generator][Arroyo Emacs =init.el= generator:1]]
 (defun arroyo-emacs-generate-init ()
-  "Write an emacs `init.el' to `arroyo-emacs-init-location' using
-   tangled files spread throughout an org-roam wiki."
  (interactive)
-  (arroyo-emacs-update-db)
-  (let ((dest arroyo-emacs-init-location)
-        (files (arroyo-emacs--ordered-files)))
-    (with-current-buffer (find-file-noselect dest)
-      (erase-buffer)
-      (dolist (file files)
-        (insert-file-contents file)
-        (goto-char (point-max)))
-      (save-buffer))))
+  ;; (setenv "ARCOLOGY_DB_PATH" "/home/rrix/org/arcology-django/db.sqlite3")
+  ;; (shell-command-to-string "nix run path:/home/rrix/org/arcology-django#arcology -- ingest_files ~/org &>/dev/null")
+  ;; (shell-command-to-string (format "nix run path:/home/rrix/org/arcology-django#arcology -- generate -m emacs -r server -d %s" arroyo-emacs-init-location))
+  (arroyo-generate-imports "emacs" nil arroyo-emacs-init-location)
+  (find-file arroyo-emacs-init-location))
+
+;; (defun arroyo-emacs-generate-init ()
+;;   "Write an emacs `init.el' to `arroyo-emacs-init-location' using
+;;    tangled files spread throughout an org-roam wiki."
+;;   (interactive)
+;;   (arroyo-emacs-update-db)
+;;   (let ((dest arroyo-emacs-init-location)
+;;         (files (arroyo-emacs--ordered-files)))
+;;     (with-current-buffer (find-file-noselect dest)
+;;       (erase-buffer)
+;;       (dolist (file files)
+;;         (insert-file-contents file)
+;;         (goto-char (point-max)))
+;;       (save-buffer))))

 (defun arroyo-emacs--module-pair-for-rel-file (unordered rel-file)
  (second (-find (lambda (elt)
@ -103,6 +111,6 @@
 (arroyo-emacs-epkgs)
 ;; generate_epkg_overrides ends here

-;; [[file:arroyo-emacs.org::*Footnotes][Footnotes:1]]
+;; [[file:arroyo-emacs.org::*need a reverse of =ARROYO_MODULE_WANTS= for specifying "wanted-by"s][need a reverse of =ARROYO_MODULE_WANTS= for specifying "wanted-by"s:1]]
 (provide 'arroyo-emacs)
-;; Footnotes:1 ends here
+;; need a reverse of =ARROYO_MODULE_WANTS= for specifying "wanted-by"s:1 ends here
--- a/arroyo-emacs.org
+++ b/arroyo-emacs.org
@ -81,22 +81,30 @@ Users of the Arroyo Home Manager generator will have access to features availabl
 :ID:       arroyo/emacs/init.el
 :END:

-=arroyo-emacs-generate-init= can be called interactively[fn:1] to assemble an =init.el= file which will have the dependencies ordered in such a way as to allow the user to user Emacs.
+=arroyo-emacs-generate-init= can be called interactively[fn:1:that is, by pressing =Alt-x= / =M-x= and then typing the name of the command; [[id:modern_interface_terms][Modern Interface Terms]]] to assemble an =init.el= file which will have the dependencies ordered in such a way as to allow the user to user Emacs.

 #+begin_src emacs-lisp :results none -r
 (defun arroyo-emacs-generate-init ()
-  "Write an emacs `init.el' to `arroyo-emacs-init-location' using
-   tangled files spread throughout an org-roam wiki."
  (interactive)
-  (arroyo-emacs-update-db)
-  (let ((dest arroyo-emacs-init-location)
-        (files (arroyo-emacs--ordered-files)))
-    (with-current-buffer (find-file-noselect dest)
-      (erase-buffer)
-      (dolist (file files)
-        (insert-file-contents file)
-        (goto-char (point-max)))
-      (save-buffer))))
+  ;; (setenv "ARCOLOGY_DB_PATH" "/home/rrix/org/arcology-django/db.sqlite3")
+  ;; (shell-command-to-string "nix run path:/home/rrix/org/arcology-django#arcology -- ingest_files ~/org &>/dev/null")
+  ;; (shell-command-to-string (format "nix run path:/home/rrix/org/arcology-django#arcology -- generate -m emacs -r server -d %s" arroyo-emacs-init-location))
+  (arroyo-generate-imports "emacs" nil arroyo-emacs-init-location)
+  (find-file arroyo-emacs-init-location))
+
+;; (defun arroyo-emacs-generate-init ()
+;;   "Write an emacs `init.el' to `arroyo-emacs-init-location' using
+;;    tangled files spread throughout an org-roam wiki."
+;;   (interactive)
+;;   (arroyo-emacs-update-db)
+;;   (let ((dest arroyo-emacs-init-location)
+;;         (files (arroyo-emacs--ordered-files)))
+;;     (with-current-buffer (find-file-noselect dest)
+;;       (erase-buffer)
+;;       (dolist (file files)
+;;         (insert-file-contents file)
+;;         (goto-char (point-max)))
+;;       (save-buffer))))

 (defun arroyo-emacs--module-pair-for-rel-file (unordered rel-file)
  (second (-find (lambda (elt)
@ -190,7 +198,7 @@ in pkgs.emacsWithPackagesFromUsePackage {

 #+ARROYO_HOME_MODULE: hm/emacs.nix
 #+ARROYO_NIXOS_MODULE: nixos/emacs.nix
-#+ARROYO_NIXOS_EXCLUDE: waterboy
+#+ARROYO_SYSTEM_EXCLUDE: waterboy

 #+begin_src nix :tangle ~/arroyo-nix/nixos/emacs.nix
 let emacsOverlay = (import ../versions.nix {}).emacsOverlay null;
@ -292,10 +300,7 @@ can use [[id:cce/org-roam][org-roam]]'s [[id:20211203T142617.812313][Arcology.Ro
 :LOGBOOK:
 - State "DONE"       from "NEXT"       [2021-08-24 Tue 19:12]
 :END:
-* Footnotes

 #+begin_src emacs-lisp 
 (provide 'arroyo-emacs)
 #+end_src
-
-[fn:1] that is, by pressing =Alt-x= / =M-x= and then typing the name of the command; [[id:modern_interface_terms][Modern Interface Terms]]
--- a/arroyo-home-manager.el
+++ b/arroyo-home-manager.el
@ -1,4 +1,3 @@
-;; [[file:arroyo-home-manager.org::+begin_src emacs-lisp][No heading:1]]
 (add-to-list 'arroyo-db-keywords "ARROYO_HOME_MODULE" nil #'equal)
 (add-to-list 'arroyo-db-keywords "CCE_HOME_MODULE" nil #'equal)
 (add-to-list 'arroyo-db--schemata
@ -7,9 +6,7 @@
                ;(file-hash   :not-null)
                (home-module :not-null)
                role]))
-;; No heading:1 ends here

-;; [[file:arroyo-home-manager.org::+begin_src emacs-lisp][No heading:2]]
 (defun arroyo-home-manager-update-db (&optional keywords-too)
  (interactive "P")
  (when keywords-too
@ -30,55 +27,61 @@
              (results (apply #'arroyo-db-query args))
              ;; attach role
              (results (-map (pcase-lambda (`(,file ,value))
-                               (list file value (arroyo-db-get "ARROYO_NIXOS_ROLE" file)))
+                               (list file value (arroyo-db-get "ARROYO_SYSTEM_ROLE" file)))
                             results)))
    (arroyo-db-query [:insert :into home-manager :values $v1]
                     (-map (lambda (result) (apply #'vector result))
                           results))))
 (add-to-list 'arroyo-db-update-functions #'arroyo-home-manager-update-db)
-;; No heading:2 ends here

-;; [[file:arroyo-home-manager.org::arroyo-home-manager-imports][arroyo-home-manager-imports]]
 (defun arroyo-home-manager-imports1 ()
  (->> [:select [home-module file] :from home-manager]
       (arroyo-db-query)))

 (defun arroyo-home-manager-imports (&optional role)
-  (if-let* ((imports (arroyo-home-manager-imports1))
-            (no-role? (not role)))
-      (->> imports
-           (-map #'car)
-           (-sort #'s-less?)
-           (s-join "\n"))
-    (progn
-      (let* ((all-files (-map #'cadr imports))
-             (role-only-files (-map #'car (arroyo-db-query [:select file :from home-manager :where (like role $r1)] (format "%%%s%%" role))))
-             (exclude-these-files (arroyo-db-by-keyword "ARROYO_NIXOS_EXCLUDE" role))
-             ;; also need to exclude things whose ARROYO_NIXOS_ROLE is not `role'!
-             (not-my-role-files
-              (-map #'car
-                    (arroyo-db-query [:select file :from home-manager
-                                      :where (!= role $s1)]
-                                     (list role))))
-             ;; combine all those lists in to a single authoritative list.
-             (amalgamated-files
-              (-union role-only-files
-                      (->> all-files
-                           (--remove (-contains? exclude-these-files it))
-                           (--remove (-contains? not-my-role-files it)))))
-             ;; make them relative to org-roam-directory for the module orderer
-             (rel-files (--map (file-relative-name it org-roam-directory) amalgamated-files)))
-        ;; filter imports
-        (->> rel-files
-             (arroyo-utils-order-modules)
-             (-map (lambda (file)
-                         (caar (arroyo-db-query [:select home-module :from home-manager
-                                                 :where (= file $s1)]
-                                                (expand-file-name file org-roam-directory)))))
-             (-filter #'identity)
-             (-sort #'s-less?))))))
-;; arroyo-home-manager-imports ends here
+  (setenv "ARCOLOGY_DB_PATH" "/home/rrix/org/arcology-django/db.sqlite3")
+  (shell-command-to-string "nix run path:/home/rrix/org/arcology-django#arcology -- ingest_files ~/org &>/dev/null")
+  (shell-command-to-string (s-join " " `("nix run path:/home/rrix/org/arcology-django#arcology"
+                                         "--"
+                                         "generate -m home-manager"
+                                         ,(when role
+                                            (format "-r %s" role))
+                                         "2>/dev/null"
+                                         " | sort"))))
+
+;; (defun *arroyo-home-manager-imports (&optional role)
+;;   (if-let* ((imports (arroyo-home-manager-imports1))
+;;             (no-role? (not role)))
+;;       (->> imports
+;;            (-map #'car)
+;;            (-sort #'s-less?)
+;;            (s-join "\n"))
+;;     (progn
+;;       (let* ((all-files (-map #'cadr imports))
+;;              (role-only-files (-map #'car (arroyo-db-query [:select file :from home-manager :where (like role $r1)] (format "%%%s%%" role))))
+;;              (exclude-these-files (arroyo-db-by-keyword "ARROYO_SYSTEM_EXCLUDE" role))
+;;              ;; also need to exclude things whose ARROYO_SYSTEM_ROLE is not `role'!
+;;              (not-my-role-files
+;;               (-map #'car
+;;                     (arroyo-db-query [:select file :from home-manager
+;;                                       :where (!= role $s1)]
+;;                                      (list role))))
+;;              ;; combine all those lists in to a single authoritative list.
+;;              (amalgamated-files
+;;               (-union role-only-files
+;;                       (->> all-files
+;;                            (--remove (-contains? exclude-these-files it))
+;;                            (--remove (-contains? not-my-role-files it)))))
+;;              ;; make them relative to org-roam-directory for the module orderer
+;;              (rel-files (--map (file-relative-name it org-roam-directory) amalgamated-files)))
+;;         ;; filter imports
+;;         (->> rel-files
+;;              (arroyo-utils-order-modules)
+;;              (-map (lambda (file)
+;;                          (caar (arroyo-db-query [:select home-module :from home-manager
+;;                                                  :where (= file $s1)]
+;;                                                 (expand-file-name file org-roam-directory)))))
+;;              (-filter #'identity)
+;;              (-sort #'s-less?))))))

-;; [[file:arroyo-home-manager.org::+begin_src emacs-lisp][No heading:4]]
 (provide 'arroyo-home-manager)
-;; No heading:4 ends here
--- a/arroyo-home-manager.org
+++ b/arroyo-home-manager.org
@ -46,7 +46,7 @@ Arroyo can construct a configuration for [[https://github.com/nix-community/home
              (results (apply #'arroyo-db-query args))
              ;; attach role
              (results (-map (pcase-lambda (`(,file ,value))
-                               (list file value (arroyo-db-get "ARROYO_NIXOS_ROLE" file)))
+                               (list file value (arroyo-db-get "ARROYO_SYSTEM_ROLE" file)))
                             results)))
    (arroyo-db-query [:insert :into home-manager :values $v1]
                     (-map (lambda (result) (apply #'vector result))
@ -63,39 +63,50 @@ Arroyo can construct a configuration for [[https://github.com/nix-community/home
       (arroyo-db-query)))

 (defun arroyo-home-manager-imports (&optional role)
-  (if-let* ((imports (arroyo-home-manager-imports1))
-            (no-role? (not role)))
-      (->> imports
-           (-map #'car)
-           (-sort #'s-less?)
-           (s-join "\n"))
-    (progn
-      (let* ((all-files (-map #'cadr imports))
-             (role-only-files (-map #'car (arroyo-db-query [:select file :from home-manager :where (like role $r1)] (format "%%%s%%" role))))
-             (exclude-these-files (arroyo-db-by-keyword "ARROYO_NIXOS_EXCLUDE" role))
-             ;; also need to exclude things whose ARROYO_NIXOS_ROLE is not `role'!
-             (not-my-role-files
-              (-map #'car
-                    (arroyo-db-query [:select file :from home-manager
-                                      :where (!= role $s1)]
-                                     (list role))))
-             ;; combine all those lists in to a single authoritative list.
-             (amalgamated-files
-              (-union role-only-files
-                      (->> all-files
-                           (--remove (-contains? exclude-these-files it))
-                           (--remove (-contains? not-my-role-files it)))))
-             ;; make them relative to org-roam-directory for the module orderer
-             (rel-files (--map (file-relative-name it org-roam-directory) amalgamated-files)))
-        ;; filter imports
-        (->> rel-files
-             (arroyo-utils-order-modules)
-             (-map (lambda (file)
-                         (caar (arroyo-db-query [:select home-module :from home-manager
-                                                 :where (= file $s1)]
-                                                (expand-file-name file org-roam-directory)))))
-             (-filter #'identity)
-             (-sort #'s-less?))))))
+  (setenv "ARCOLOGY_DB_PATH" "/home/rrix/org/arcology-django/db.sqlite3")
+  (shell-command-to-string "nix run path:/home/rrix/org/arcology-django#arcology -- ingest_files ~/org &>/dev/null")
+  (shell-command-to-string (s-join " " `("nix run path:/home/rrix/org/arcology-django#arcology"
+                                         "--"
+                                         "generate -m home-manager"
+                                         ,(when role
+                                            (format "-r %s" role))
+                                         "2>/dev/null"
+                                         " | sort"))))
+
+;; (defun *arroyo-home-manager-imports (&optional role)
+;;   (if-let* ((imports (arroyo-home-manager-imports1))
+;;             (no-role? (not role)))
+;;       (->> imports
+;;            (-map #'car)
+;;            (-sort #'s-less?)
+;;            (s-join "\n"))
+;;     (progn
+;;       (let* ((all-files (-map #'cadr imports))
+;;              (role-only-files (-map #'car (arroyo-db-query [:select file :from home-manager :where (like role $r1)] (format "%%%s%%" role))))
+;;              (exclude-these-files (arroyo-db-by-keyword "ARROYO_SYSTEM_EXCLUDE" role))
+;;              ;; also need to exclude things whose ARROYO_SYSTEM_ROLE is not `role'!
+;;              (not-my-role-files
+;;               (-map #'car
+;;                     (arroyo-db-query [:select file :from home-manager
+;;                                       :where (!= role $s1)]
+;;                                      (list role))))
+;;              ;; combine all those lists in to a single authoritative list.
+;;              (amalgamated-files
+;;               (-union role-only-files
+;;                       (->> all-files
+;;                            (--remove (-contains? exclude-these-files it))
+;;                            (--remove (-contains? not-my-role-files it)))))
+;;              ;; make them relative to org-roam-directory for the module orderer
+;;              (rel-files (--map (file-relative-name it org-roam-directory) amalgamated-files)))
+;;         ;; filter imports
+;;         (->> rel-files
+;;              (arroyo-utils-order-modules)
+;;              (-map (lambda (file)
+;;                          (caar (arroyo-db-query [:select home-module :from home-manager
+;;                                                  :where (= file $s1)]
+;;                                                 (expand-file-name file org-roam-directory)))))
+;;              (-filter #'identity)
+;;              (-sort #'s-less?))))))
 #+end_src

 #+begin_src emacs-lisp
--- a/arroyo-native-parser.org
+++ b/arroyo-native-parser.org
--- a/arroyo-nix.org
+++ b/arroyo-nix.org
@ -50,6 +50,9 @@ in {
  lib = import ./lib { inherit pkgs; }; # functions
  # modules = import ./modules; # NixOS modules
  # overlays = import ./overlays; # nixpkgs overlays
+  overlays = [
+    (import ./overlay.nix)
+  ];

  nixGL = callPackage ./pkgs/nixgl.nix {};
  myEmacs = callPackage ./pkgs/emacs.nix {};
@ -85,7 +88,7 @@ in {
  qasync = toPythonModule (callPackage ./pkgs/qasync.nix {});
  svgelements = toPythonModule (callPackage ./pkgs/svgelements.nix {});
  hnswlib = toPythonModule (callPackage ./pkgs/hnswlib.nix {});
-  watchfiles = toPythonModule (callPackage ./pkgs/watchfiles/default.nix {});
+  # watchfiles = toPythonModule (callPackage ./pkgs/watchfiles/default.nix {});

  feediverse = toPythonModule (callPackage ./pkgs/feediverse.nix {});

--- a/arroyo-nixos.el
+++ b/arroyo-nixos.el
@ -4,8 +4,8 @@
  :type 'string)

 (dolist (mod '("ARROYO_NIXOS_MODULE"
-               "ARROYO_NIXOS_ROLE"
-               "ARROYO_NIXOS_EXCLUDE"))
+               "ARROYO_SYSTEM_ROLE"
+               "ARROYO_SYSTEM_EXCLUDE"))
  (add-to-list 'arroyo-db-keywords mod nil #'equal))

 (add-to-list 'arroyo-db--schemata
@ -33,7 +33,7 @@
                       "ARROYO_NIXOS_MODULE")))
              (results (apply #'arroyo-db-query args))
              (results (-map (pcase-lambda (`(,file ,value))
-                               (list file value (arroyo-db-get "ARROYO_NIXOS_ROLE" file)))
+                               (list file value (arroyo-db-get "ARROYO_SYSTEM_ROLE" file)))
                             results)))
    (arroyo-db-query [:insert :into nixos :values $v1]
                     (-map (lambda (result) (apply #'vector result))
@ -47,9 +47,9 @@
      (-map #'car imports)
    (progn
      (let* ((all-files (-map #'cadr imports))
-             (role-only-files (arroyo-db-by-keyword "ARROYO_NIXOS_ROLE" role))
-             (exclude-these-files (arroyo-db-by-keyword "ARROYO_NIXOS_EXCLUDE" role))
-             ;; also need to exclude things whose ARROYO_NIXOS_ROLE is not `role'!
+             (role-only-files (arroyo-db-by-keyword "ARROYO_SYSTEM_ROLE" role))
+             (exclude-these-files (arroyo-db-by-keyword "ARROYO_SYSTEM_EXCLUDE" role))
+             ;; also need to exclude things whose ARROYO_SYSTEM_ROLE is not `role'!
             (not-my-role-files
              (-map #'car
                    (arroyo-db-query [:select file :from nixos
--- a/arroyo-nixos.org
+++ b/arroyo-nixos.org
@ -2,7 +2,6 @@
 :ID:       arroyo/nixos
 :END:
 #+TITLE: Arroyo NixOS Generator
-#+filetags: :Project:
 #+PROPERTY: header-args            :mkdirp yes
 #+PROPERTY: header-args:emacs-lisp :tangle arroyo-nixos.el
 #+AUTO_TANGLE: t
@ -24,8 +23,8 @@ This thing is pretty functionally similar to the [[id:arroyo/home-manager][Arroy
  :type 'string)

 (dolist (mod '("ARROYO_NIXOS_MODULE"
-               "ARROYO_NIXOS_ROLE"
-               "ARROYO_NIXOS_EXCLUDE"))
+               "ARROYO_SYSTEM_ROLE"
+               "ARROYO_SYSTEM_EXCLUDE"))
  (add-to-list 'arroyo-db-keywords mod nil #'equal))

 (add-to-list 'arroyo-db--schemata
@ -55,7 +54,7 @@ This thing is pretty functionally similar to the [[id:arroyo/home-manager][Arroy
                       "ARROYO_NIXOS_MODULE")))
              (results (apply #'arroyo-db-query args))
              (results (-map (pcase-lambda (`(,file ,value))
-                               (list file value (arroyo-db-get "ARROYO_NIXOS_ROLE" file)))
+                               (list file value (arroyo-db-get "ARROYO_SYSTEM_ROLE" file)))
                             results)))
    (arroyo-db-query [:insert :into nixos :values $v1]
                     (-map (lambda (result) (apply #'vector result))
@ -74,9 +73,9 @@ This thing is pretty functionally similar to the [[id:arroyo/home-manager][Arroy
      (-map #'car imports)
    (progn
      (let* ((all-files (-map #'cadr imports))
-             (role-only-files (arroyo-db-by-keyword "ARROYO_NIXOS_ROLE" role))
-             (exclude-these-files (arroyo-db-by-keyword "ARROYO_NIXOS_EXCLUDE" role))
-             ;; also need to exclude things whose ARROYO_NIXOS_ROLE is not `role'!
+             (role-only-files (arroyo-db-by-keyword "ARROYO_SYSTEM_ROLE" role))
+             (exclude-these-files (arroyo-db-by-keyword "ARROYO_SYSTEM_EXCLUDE" role))
+             ;; also need to exclude things whose ARROYO_ARROYO_SYSTEM_EXCLUDErole'!
             (not-my-role-files
              (-map #'car
                    (arroyo-db-query [:select file :from nixos
--- a/arroyo-system-cache.org
+++ b/arroyo-system-cache.org
@ -16,7 +16,7 @@ Most of this exists in one form or another, I plan to take heavy inspiration fro

 * headmatter & db functions

-#+begin_src emacs-lisp 
+#+begin_src emacs-lisp
 (defcustom arroyo-db-location (expand-file-name "arroyo.db" user-emacs-directory)
  "Arroyo System Cache location"
  :group 'arroyo)
@ -177,13 +177,13 @@ SQL can be either the emacsql vector representation, or a string."

 (defun arroyo--files-for-role (role)
  (->> role
-       (arroyo-db-by-keyword "ARROYO_NIXOS_ROLE") 
+       (arroyo-db-by-keyword "ARROYO_SYSTEM_ROLE")
       (--filter (arroyo-db-get "ARROYO_NIXOS_MODULE" it))
       (--map (cons it (arroyo-db--get-file-title-from-org-roam it)))))

 (defun arroyo-db--get-file-title-from-org-roam (file)
  (->> file
-       (org-roam-db-query [:select title :from nodes :where (= level 0) :and (= file $s1)]) 
+       (org-roam-db-query [:select title :from nodes :where (= level 0) :and (= file $s1)])
       (caar)))
 #+end_src

@ -227,7 +227,7 @@ auto-tangle files that are modified?

 This =keywords= table provides a simple key-value store which can be further derived in to tables for smarter applications like the [[id:arroyo/emacs][Arroyo Emacs Generator]], etc. Included here is a [[id:d1d90d34-e7ee-47b0-b985-4463e4f538d2][hash function]].

-#+begin_src emacs-lisp 
+#+begin_src emacs-lisp
 (defcustom arroyo-db-keywords '("BIRTHDAY" "PRONOUNS" "LOCATION")
  "Keyword properties which will be stored in the org-roam db keywords table."
  :type '(repeat string)
@ -245,7 +245,7 @@ It's a simple file-level-properties key/value table shaped like:
             '(keywords
               [(file :not-null)
                (keyword :not-null)
-                (value :not-null)])) 
+                (value :not-null)]))
 #+end_src

 And we cache the files' hash as well to compare against the org-roam database.
@ -254,7 +254,7 @@ And we cache the files' hash as well to compare against the org-roam database.
 (add-to-list 'arroyo-db--schemata
             '(files
               [(file :not-null)
-                (hash :not-null)])) 
+                (hash :not-null)]))
 #+end_src

 #+begin_src emacs-lisp
@ -305,7 +305,7 @@ Return the number of rows inserted."

 functions from org-roam v1 to extract file-level properties

-#+begin_src emacs-lisp 
+#+begin_src emacs-lisp
 (defun arroyo-db--collect-keywords (keywords)
  "Collect all Org KEYWORDS in the current buffer."
  (if (functionp 'org-collect-keywords)
@ -335,7 +335,7 @@ functions from org-roam v1 to extract file-level properties
    (org-with-point-at 1
      (dolist (prop props ret)
        (when-let ((v (org-entry-get (point) prop)))
-          (push (cons prop v) ret)))))) 
+          (push (cons prop v) ret))))))

 (defun arroyo-db--extract-global-props (props)
  "Extract PROPS from the current Org buffer.
@ -343,7 +343,7 @@ Props are extracted from both the file-level property drawer (if
 any), and Org keywords. Org keywords take precedence."
  (append
   (arroyo-db--extract-global-props-keyword props)
-   (arroyo-db--extract-global-props-drawer props))) 
+   (arroyo-db--extract-global-props-drawer props)))
 #+end_src

 * Footmatter
--- a/arroyo/init.py
+++ b/arroyo/init.py
@ -0,0 +1,4 @@
+# [[file:../arroyo-native-parser.org::*Python Package][Python Package:1]]
+from .arroyo_rs import parse_file, InvalidDocError
+from .arroyo_rs import htmlize_file, ExportOptions
+# Python Package:1 ends here
--- a/arroyo/main.py
+++ b/arroyo/main.py
@ -0,0 +1,54 @@
+# [[file:../arroyo-native-parser.org::*Click command wrapper][Click command wrapper:1]]
+import os
+import click
+import glob
+
+# from . import persist_one_file
+from .arroyo_rs import htmlize_file, ExportOptions
+# from . import models
+# from sqlmodel import Session
+# Click command wrapper:1 ends here
+
+# [[file:../arroyo-native-parser.org::*Click command wrapper][Click command wrapper:2]]
+@click.group()
+def cli():
+  pass
+
+@cli.command()
+@click.option("--source", "-s", help="Org source directory", default="~/org")
+@click.option("--file-glob", "-g", help="File search glob", default="**/*.org")
+@click.option("--dest", "-d", help="Sqlite Database Location", default="./arroyo.db")
+def generate_db(source, dest, file_glob):
+  #engine = models.make_engine(dest)
+
+  #expanded_src = os.path.expanduser(source)
+  #files = glob.glob(file_glob, root_dir=expanded_src, recursive=True)
+  #files = map(lambda it: os.path.join(expanded_src, it), files)
+  #files = list(filter(os.path.isfile, files))
+
+  #with Session(engine) as session:
+  #  docs = [
+  #    x for x in [
+  #      persist_one_file(session, path)
+  #      for path in files
+  #    ] if x is not None
+  #  ]
+
+  print(f"Parsed {len(files)} files.")
+  print(f"Persisted {len(docs)} docs.")
+
+
+@cli.command()
+@click.option("--file", "-f", help="The file to export")
+def export_document(file):
+  # in The Real World this is loaded from DB and generated.
+  options = ExportOptions(
+    link_retargets = {"currently_reading": "https://rix.si/hello-world"}
+  )
+  print(htmlize_file(file, options))
+# Click command wrapper:2 ends here
+
+# [[file:../arroyo-native-parser.org::*Click command wrapper][Click command wrapper:3]]
+if __name__ == "__main__":
+  cli()
+# Click command wrapper:3 ends here
--- a/arroyo_rs.org
+++ b/arroyo_rs.org
@ -0,0 +1,31 @@
+:PROPERTIES:
+:ID:       20231021T165413.872542
+:ROAM_ALIASES: "Arroyo Org Parser" arroyo-rs
+:ROAM_REFS: https://code.rix.si/rrix/arroyo_rs
+:END:
+#+TITLE: arroyo-rs is an org-mode document parser for Python and Rust
+#+FILETAGS: :Project:
+
+=arroyo-rs= is an [[https://orgmode.org/][org-mode]] parser library for [[https://engine.arcology.garden][The Arcology Project]]. It exposes a simple interface to the [[https://www.rust-lang.org/][Rust]] [[https://docs.rs/orgize/][Orgize]] library, wrapped in [[https://pyo3.rs/][pyo3]] bindings to be accessible from [[https://python.org][Python]]. Most of the Org-mode parsers I've looked at so far lack certain features or parsing capabilities that would make it difficult or impossible to implement [[https://cce.whatthefuck.computer/arroyo/][Arroyo]] and the Arcology Project.
+
+This package exports two public functions and a handful of types:
+
+- =arroyo_rs.parse_file(path: str) -> Document=
+- =arroyo_rs.export_html(path: str) -> str= (This hasn't been implemented yet and the shape of it might change, for example adding DB-backed link rewriting for the Arcology)
+- and types which map to the Arcology Project's database types.
+
+* Further Work                                                        :Tasks:
+
+** NEXT pull =arroyo_rs= in to literate docs
+
+** NEXT evaluate moving [[id:arcology/arroyo-page][Arroyo Arcology Generator]] in to this repo
+
+- SQLModel python stuff for DB persistence
+
+** NEXT move keyword and file table generation to arroyo_rs
+
+** NEXT move the emacs, nixos, and home-manager generators across
+
+** NEXT source code block extraction & babel execution?
+
+how wild would it be to have a little wasm environment in the rust to execute code blocks? at the very least i want to build a tangler and some Python modules to populate Arroyo code generators.
--- a/default.nix
+++ b/default.nix
@ -0,0 +1,42 @@
+# [[file:arroyo-native-parser.org::*Python package built with =maturin=][Python package built with =maturin=:1]]
+{
+  pkgs ? import <nixpkgs> {},
+  lib  ? pkgs.lib,
+
+  python3,
+}:
+
+python3.pkgs.buildPythonPackage rec {
+  pname = "arroyo_rs";
+  version = "0.0.1";
+  format = "pyproject";
+
+  src = ./.;
+
+  nativeBuildInputs = with pkgs; [
+    cargo rustc
+  ] ++ (with pkgs.rustPlatform; [
+    maturinBuildHook
+    cargoSetupHook
+  ]);
+
+  propagatedBuildInputs = with pkgs; [
+    python3.pkgs.click
+  ];
+
+  cargoDeps = pkgs.rustPlatform.importCargoLock {
+    lockFile = ./Cargo.lock;
+
+    outputHashes = {
+      "orgize-0.9.0" = "sha256-Nn7+nQVBn2Gn0+uHvlY8NKSV/bPVEQK9sFPTzTAcDWY=";
+    };
+  };
+
+  meta = with lib; {
+    description = "An Org-Mode parser library for the arcology project";
+    homepage = "https://cce.whatthefuck.computer/arroyo";
+    license = licenses.unfree;
+    maintainers = with maintainers; [ rrix ];
+  };
+}
+# Python package built with =maturin=:1 ends here
--- a/flake.lock
+++ b/flake.lock
@ -0,0 +1,59 @@
+{
+  "nodes": {
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1694529238,
+        "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "ff7b65b44d01cf9ba6a71320833626af21126384",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1698553279,
+        "narHash": "sha256-T/9P8yBSLcqo/v+FTOBK+0rjzjPMctVymZydbvR/Fak=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "90e85bc7c1a6fc0760a94ace129d3a1c61c3d035",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "type": "indirect"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}
--- a/flake.nix
+++ b/flake.nix
@ -0,0 +1,35 @@
+# [[file:arroyo-native-parser.org::*Put it all together and make it distributable with a Nix flake][Put it all together and make it distributable with a Nix flake:1]]
+{
+  description = "Arroyo Parser Exporter Library";
+
+  inputs.flake-utils.url = "github:numtide/flake-utils";
+
+  outputs = { self, nixpkgs, flake-utils }:
+    flake-utils.lib.eachDefaultSystem (system:
+      let
+        pkgs = import nixpkgs {
+          inherit system;
+          config.allowUnfree = true;
+        };
+
+        python3 = pkgs.python3;
+      in
+        {
+          devShells.default = pkgs.callPackage ./shell.nix { inherit python3; };
+
+          packages = rec {
+            arroyo_rs = pkgs.callPackage ./default.nix { inherit python3; };
+            default = arroyo_rs;
+          };
+
+          apps = rec {
+            arroyo_rs = flake-utils.lib.mkApp {
+              drv = self.packages.${system}.arroyo_rs;
+              exePath = "/bin/arroyo";
+            };
+            default = arroyo_rs;
+          };
+        }
+    );
+}
+# Put it all together and make it distributable with a Nix flake:1 ends here
--- a/pyproject.toml
+++ b/pyproject.toml
@ -0,0 +1,24 @@
+# [[file:arroyo-native-parser.org::*Python package definition][Python package definition:1]]
+[project]
+name = "arroyo"
+version = "0.0.1"
+description = "org-mode metadata extractor"
+# license = "Hey Smell This"
+readme = "README.org"
+dependencies = ["click ~=8.1"]
+requires-python = ">=3.10"
+authors = [
+    { name = "Ryan Rix", email = "code@whatthefuck.computer" }
+]
+
+[project.scripts]
+arroyo = "arroyo.__main__:cli"
+
+[build-system]
+requires = ["maturin>=1.0,<2.0"]
+build-backend = "maturin"
+
+[tool.maturin]
+module-name = "arroyo.arroyo_rs"
+features = ["pyo3/extension-module"]
+# Python package definition:1 ends here
--- a/rustfmt.toml
+++ b/rustfmt.toml
@ -0,0 +1,3 @@
+# [[file:arroyo-native-parser.org::*Dev support files][Dev support files:1]]
+edition = "2021"
+# Dev support files:1 ends here
--- a/shell.nix
+++ b/shell.nix
@ -0,0 +1,34 @@
+# [[file:arroyo-native-parser.org::*Development Shell][Development Shell:1]]
+{ pkgs ? import <nixpkgs> {},
+  python3 ? pkgs.python3
+}:
+let
+  myPy = python3.withPackages( pp: with pp; [
+    pip
+    pytest
+  ]);
+in pkgs.mkShell {
+  packages = with pkgs; [
+    cargo
+    rustc
+    gcc
+
+    rust-analyzer
+    rustfmt
+    clippy
+
+    maturin
+    myPy
+    pyright
+    black
+  ];
+  RUST_SRC_PATH = "${pkgs.rust.packages.stable.rustPlatform.rustLibSrc}";
+  NIX_CONFIG = "builders =";
+
+  shellHook = ''
+    test -f venv/bin/python3 || python3 -m venv venv
+    . venv/bin/activate
+    maturin develop -j6
+  '';
+}
+# Development Shell:1 ends here
--- a/src/export.rs
+++ b/src/export.rs
@ -0,0 +1,212 @@
+// [[file:../arroyo-native-parser.org::*The HTML exporter][The HTML exporter:1]]
+use anyhow::Result;
+use pyo3::prelude::*;
+use std::collections::HashMap;
+use std::convert::From;
+use std::fs;
+use std::io::{Error, Write};
+use std::marker::PhantomData;
+
+use regex::Regex;
+
+use orgize::export::{DefaultHtmlHandler, HtmlEscape, HtmlHandler, SyntectHtmlHandler};
+use orgize::{Element, Org};
+// The HTML exporter:1 ends here
+
+// [[file:../arroyo-native-parser.org::*The HTML exporter][The HTML exporter:2]]
+#[derive(Default, Debug, Clone)]
+#[pyclass(dict)]
+pub struct ExportOptions {
+    /// id:{the_id} -> URL rewrites
+    #[pyo3(get)]
+    pub link_retargets: HashMap<String, String>,
+}
+
+#[pymethods]
+impl ExportOptions {
+    #[new]
+    fn new(link_retargets: HashMap<String, String>) -> Self {
+        ExportOptions {
+            link_retargets,
+            ..Default::default()
+        }
+    }
+
+    // add these later if i feel like it
+    // pub fn __repr__(slf: PyRef<'_, Self>) -> PyResult<String> {
+    //     Ok(slf.to_string())
+    // }
+
+    // pub fn __str__(slf: PyRef<'_, Self>) -> PyResult<String> {
+    //     Self::__repr__(slf)
+    // }
+}
+// The HTML exporter:2 ends here
+
+// [[file:../arroyo-native-parser.org::*The HTML exporter][The HTML exporter:3]]
+pub struct ArroyoHtmlHandler<E: From<Error>, H: HtmlHandler<E>> {
+    pub options: ExportOptions,
+    /// inner html handler
+    pub inner: H,
+    /// handler error type
+    pub error_type: PhantomData<E>,
+    /// file-property drawer state tracking
+    in_drawer: bool,
+}
+
+impl<E: From<Error>, H: HtmlHandler<E>> ArroyoHtmlHandler<E, H> {
+    pub fn new(options: ExportOptions, inner: H) -> Self {
+        ArroyoHtmlHandler {
+            inner,
+            options,
+            ..Default::default()
+        }
+    }
+}
+
+impl<E: From<Error>, H: HtmlHandler<E>> Default for ArroyoHtmlHandler<E, H> {
+    fn default() -> Self {
+        ArroyoHtmlHandler {
+            inner: H::default(),
+            error_type: PhantomData,
+            in_drawer: false,
+            options: ExportOptions::default(),
+        }
+    }
+}
+// The HTML exporter:3 ends here
+
+// [[file:../arroyo-native-parser.org::*The Custom HTML Exporter Extensions][The Custom HTML Exporter Extensions:1]]
+impl<E: From<Error>, H: HtmlHandler<E>> HtmlHandler<E> for ArroyoHtmlHandler<E, H> {
+    fn start<W: Write>(&mut self, mut w: W, element: &Element) -> Result<(), E> {
+        if self.in_drawer {
+            return Ok(());
+        }
+        match element {
+// The Custom HTML Exporter Extensions:1 ends here
+
+// [[file:../arroyo-native-parser.org::*The Custom HTML Exporter Extensions][The Custom HTML Exporter Extensions:2]]
+            Element::Drawer(drawer) => {
+                self.in_drawer = drawer.name == "PROPERTIES" || drawer.name == "REVIEW_DATA"
+            }
+// The Custom HTML Exporter Extensions:2 ends here
+
+// [[file:../arroyo-native-parser.org::*The Custom HTML Exporter Extensions][The Custom HTML Exporter Extensions:3]]
+            Element::Text { value: before } => {
+                let re = Regex::new(
+                    r"(?x)
+                          \{\{([^\}]+)}     # grab the answer
+                          (?:\{([^\}]+?)})? # grab a hint
+                          @[0-9]+}          # exclude the question number
+                        ",
+                )
+                .unwrap();
+                let after = re.replace_all(before, "<span class='fc-cloze' title='$2'>$1</span>");
+                if after.eq(before) {
+                    self.inner.start(w, &Element::Text { value: after })?
+                } else {
+                    write!(w, "{}", after)?
+                }
+            }
+// The Custom HTML Exporter Extensions:3 ends here
+
+// [[file:../arroyo-native-parser.org::*The Custom HTML Exporter Extensions][The Custom HTML Exporter Extensions:4]]
+            Element::Link(link) => {
+                let string_path = link.path.to_string();
+                let (proto, stripped_dest) = match string_path.split_once(':') {
+                    Some((proto, stripped_dest)) => (proto, stripped_dest.into()),
+                    None => ("", string_path),
+                };
+                match proto {
+                    "id" => {
+                        let maybe_new_target = self.options.link_retargets.get(&stripped_dest);
+                        match maybe_new_target {
+                            Some(path) => {
+                                let desc = link.desc.clone().unwrap_or(path.clone().into());
+                                write!(
+                                    w,
+                                    "<a class=\"internal\" href=\"{}\">{}</a>",
+                                    HtmlEscape(&path),
+                                    HtmlEscape(&desc),
+                                )?
+                            }
+                            _ => {
+                                let desc = link.desc.clone().unwrap_or(link.path.clone());
+                                write!(
+                                    w,
+                                    "<a href=\"404?key={}\">{}</a>",
+                                    HtmlEscape(&link.path),
+                                    HtmlEscape(&desc),
+                                )?
+                            }
+                        };
+                    }
+                    "roam" => {
+                        let desc = link.desc.clone().unwrap_or(link.path.clone());
+                        write!(
+                            w,
+                            "<a href=\"404?key={}\">{}</a>",
+                            HtmlEscape(&link.path),
+                            HtmlEscape(&desc),
+                        )?
+                    }
+                    _ => {
+                        self.inner.start(w, &Element::Link(link.clone()))?
+                    }
+                }
+            }
+// The Custom HTML Exporter Extensions:4 ends here
+
+// [[file:../arroyo-native-parser.org::*The Custom HTML Exporter Extensions][The Custom HTML Exporter Extensions:5]]
+            _ => self.inner.start(w, element)?,
+        }
+        Ok(())
+    }
+
+    fn end<W: Write>(&mut self, w: W, element: &Element) -> Result<(), E> {
+        match element {
+            // reset the drawer state tracking
+            Element::Drawer(_drawer) => {
+                self.in_drawer = false;
+            }
+            _ => {
+                if self.in_drawer {
+                    return Ok(());
+                }
+                self.inner.end(w, element)?
+            }
+        }
+        Ok(())
+    }
+}
+// The Custom HTML Exporter Extensions:5 ends here
+
+// [[file:../arroyo-native-parser.org::*The Public Interface][The Public Interface:1]]
+// sure would be nice..... some day i'll understand lifetimes enough
+// to write a function that goes path -> orgize::Org
+// use crate::parse::orgize_document;
+
+pub fn htmlize_file(path: String, options: ExportOptions) -> Result<String> {
+    let mut handler = ArroyoHtmlHandler::new(options, SyntectHtmlHandler::new(DefaultHtmlHandler));
+    let org = String::from_utf8(fs::read(path.clone())?).unwrap();
+    let org_tree = &Org::parse_custom(
+        &org,
+        &orgize::ParseConfig {
+            // Need to pull these from environment or options...
+            todo_keywords: (
+                vec![
+                    "NEXT".to_string(),
+                    "INPROGRESS".to_string(),
+                    "WAITING".to_string(),
+                ],
+                vec!["DONE".to_string(), "CANCELLED".to_string()],
+            ),
+            ..Default::default()
+        },
+    );
+
+    let mut vec = vec![];
+    org_tree.write_html_custom(&mut vec, &mut handler)?;
+    Ok(String::from_utf8(vec)?)
+}
+// The Public Interface:1 ends here
--- a/src/lib.rs
+++ b/src/lib.rs
@ -0,0 +1,30 @@
+// [[file:../arroyo-native-parser.org::*Library definition and exports for the Python library][Library definition and exports for the Python library:1]]
+use pyo3::prelude::*;
+
+pub mod parse;
+pub mod export;
+pub mod types;
+
+#[pymodule]
+fn arroyo_rs(py: Python, m: &PyModule) -> PyResult<()> {
+    #[pyfn(m)]
+    fn parse_file(path: String) -> PyResult<types::Document> {
+        Ok(parse::parse_document(path)?)
+    }
+
+    #[pyfn(m)]
+    fn htmlize_file(path: String, options: export::ExportOptions) -> PyResult<String> {
+        Ok(export::htmlize_file(path, options)?)
+    }
+
+    m.add_class::<types::Document>()?;
+    m.add_class::<types::Heading>()?;
+    m.add_class::<types::Keyword>()?;
+    m.add_class::<types::Link>()?;
+    m.add_class::<export::ExportOptions>()?;
+
+    m.add("InvalidDocError", py.get_type::<types::InvalidDocError>())?;
+
+    Ok(())
+}
+// Library definition and exports for the Python library:1 ends here
--- a/src/parse.rs
+++ b/src/parse.rs
@ -0,0 +1,370 @@
+// [[file:../arroyo-native-parser.org::*The Parser][The Parser:1]]
+use anyhow::Result;
+use itertools::Itertools;
+use lexpr;
+use orgize::Event;
+use orgize::Org;
+use std::boxed::Box;
+use std::collections::HashMap;
+use std::{error::Error, fs};
+// use std::collections::HashMap;
+
+use crate::types::{Document, Heading, InvalidDocError, Keyword, Link};
+// The Parser:1 ends here
+
+// [[file:../arroyo-native-parser.org::*The public interface][The public interface:1]]
+pub fn parse_document(path: String) -> Result<Document> {
+    let org = String::from_utf8(fs::read(path.clone())?).unwrap();
+    let org_tree = &Org::parse_custom(
+        &org,
+        &orgize::ParseConfig {
+            // Need to pull these from environment or options...
+            todo_keywords: (vec!["NEXT".to_string(), "INPROGRESS".to_string(), "WAITING".to_string()],
+                            vec!["DONE".to_string(), "CANCELLED".to_string()]),
+            ..Default::default()
+        }
+    );
+    let keywords = extract_metadata(path.clone(), org_tree)?;
+    let headings = extract_headings(path.clone(), org_tree)?;
+
+    Ok(Document {
+        path,
+        headings,
+        keywords,
+    })
+}
+// The public interface:1 ends here
+
+// [[file:../arroyo-native-parser.org::*Extracting Arroyo Keywords][Extracting Arroyo Keywords:1]]
+pub fn extract_metadata(path: String, tree: &Org) -> Result<Vec<Keyword>> {
+    Ok(tree
+        .keywords()
+        .map(|kw| Keyword {
+            file: path.clone(),
+            keyword: kw.key.to_string(),
+            value: kw.value.to_string(),
+        })
+        .collect())
+}
+// Extracting Arroyo Keywords:1 ends here
+
+// [[file:../arroyo-native-parser.org::*Extracting Arroyo Headings][Extracting Arroyo Headings:1]]
+pub fn extract_headings(path: String, tree: &Org) -> Result<Vec<Heading>> {
+// Extracting Arroyo Headings:1 ends here
+
+// [[file:../arroyo-native-parser.org::*Extracting Arroyo Headings][Extracting Arroyo Headings:2]]
+let mut in_drawer: bool = false;
+let mut id_crumbs: Vec<Option<String>> = Vec::new();
+let mut cur_id: Option<String> = None;
+let mut cur_level: usize = 0;
+let mut headings: Vec<Heading> = Vec::new();
+headings.push(Heading::default());
+let mut links: HashMap<String, Vec<Link>> = HashMap::new();
+let mut inherited_tags: Vec<Vec<String>> = Vec::new();
+// Extracting Arroyo Headings:2 ends here
+
+// [[file:../arroyo-native-parser.org::*Extracting Arroyo Headings][Extracting Arroyo Headings:3]]
+// file level metadata + filetags
+let file_metadata = extract_metadata(path.clone(), tree)?;
+let filetags = match file_metadata
+    .iter()
+    .find(|kw| kw.keyword.to_lowercase() == "filetags")
+{
+    Some(kw) => kw
+        .value
+        .split(':')
+        .map(|s| s.to_string())
+        .filter(|s| !s.is_empty())
+        .collect(),
+    _ => Vec::<String>::new(),
+};
+headings[0].tags = Some(filetags.clone());
+// Extracting Arroyo Headings:3 ends here
+
+// [[file:../arroyo-native-parser.org::*Extracting Arroyo Headings][Extracting Arroyo Headings:4]]
+// Extract document title and apply to level 0 heading
+let doc_title = match file_metadata
+    .iter()
+    .find(|kw| kw.keyword.to_lowercase() == "title")
+{
+    Some(kw) => kw.value.clone(),
+    _ => String::from(""),
+};
+headings[0].text = doc_title;
+// Extracting Arroyo Headings:4 ends here
+
+// [[file:../arroyo-native-parser.org::*Extracting Arroyo Headings][Extracting Arroyo Headings:5]]
+// state machine go brrr
+tree.iter()
+    .map(|event| {
+        match event {
+// Extracting Arroyo Headings:5 ends here
+
+// [[file:../arroyo-native-parser.org::*Heading parser][Heading parser:1]]
+Event::Start(orgize::Element::Title(title)) => {
+    let properties = title.properties.clone().into_hash_map();
+    cur_id = properties.get("ID").map(|id| id.clone().into());
+
+    id_crumbs.truncate(cur_level + 1);
+    id_crumbs.push(cur_id.clone());
+
+    let refs = properties
+        .get("ROAM_REFS")
+        .map(|s| split_quoted_string(s.to_string()).ok())
+        .unwrap_or(Some(vec![]));
+    let aliases = properties
+        .get("ROAM_ALIASES")
+        .map(|s| split_quoted_string(s.to_string()).ok())
+        .unwrap_or(Some(vec![]));
+    cur_level = title.level;
+
+    // reset the tags table
+    inherited_tags.truncate(cur_level - 1);
+    let new_tags: Vec<String> = title
+        .tags
+        .iter()
+        .map(|mbox| mbox.clone().to_string())
+        .collect();
+    inherited_tags.push(new_tags);
+
+    let most_tags = inherited_tags.concat();
+    let all_tags: Vec<String> = [filetags.clone(), most_tags].concat();
+
+    let h = Heading {
+        id: cur_id.clone(),
+        level: cur_level,
+        text: title.raw.to_string(),
+        tags: match all_tags.len() {
+            0 => None,
+            _ => Some(all_tags),
+        },
+        refs,
+        aliases,
+        ..Default::default()
+    };
+    headings.push(h);
+    Ok(())
+}
+// Heading parser:1 ends here
+
+// [[file:../arroyo-native-parser.org::*File-level Property Drawer parsing][File-level Property Drawer parsing:1]]
+Event::Start(orgize::Element::Drawer(drawer)) => {
+    in_drawer = drawer.name == "PROPERTIES" && headings[0].id.is_none();
+    Ok(())
+}
+// File-level Property Drawer parsing:1 ends here
+
+// [[file:../arroyo-native-parser.org::*File-level Property Drawer parsing][File-level Property Drawer parsing:2]]
+Event::Start(orgize::Element::Text { value }) => {
+    if in_drawer {
+        // this is where we rely on forked orgize
+        let (_, prop_drawer): (_, orgize::elements::PropertiesMap) =
+            orgize::elements::Drawer::parse_drawer_content(value)
+                .expect("failed to parse properties drawer");
+        let properties = prop_drawer.into_hash_map();
+
+        // update cur_id and heading 0 ID since this is
+        // implied to be the first drawer, but it's kind
+        // of :yikes: to think about it like that! we
+        // could be genious enough to have a floating
+        // PROPERTIES drawer that would muck things up
+        cur_id = properties.get("ID").map(|s| s.to_string());
+        if cur_id.is_none() {
+            cur_id = properties.get("CUSTOM_ID").map(|s| s.to_string())
+        }
+
+        id_crumbs = vec![cur_id.clone()];
+        headings[0].id = cur_id.clone();
+
+        headings[0].aliases = properties
+            .get("ROAM_ALIASES")
+            .map(|s| split_quoted_string(s.to_string()).ok())
+            .unwrap_or(Some(vec![]));
+        headings[0].refs = properties
+            .get("ROAM_REFS")
+            .map(|s| split_quoted_string(s.to_string()).ok())
+            .unwrap_or(Some(vec![]));
+    }
+
+    if headings[0].id.is_none() {
+        return Err(InvalidDocError::new_err(format!(
+            "Root ID is None in {}",
+            path
+        )));
+    }
+
+    Ok(())
+}
+// File-level Property Drawer parsing:2 ends here
+
+// [[file:../arroyo-native-parser.org::*File-level Property Drawer parsing][File-level Property Drawer parsing:3]]
+Event::End(orgize::Element::Drawer(_drawer)) => {
+    in_drawer = false;
+    Ok(())
+}
+// File-level Property Drawer parsing:3 ends here
+
+// [[file:../arroyo-native-parser.org::*Link parsing][Link parsing:1]]
+// Stash links outside the match block in a HashMap shape
+// of heading id -> list of links; it would be nice if the
+// match block returned an Option<Link> but that doesn't
+// play well with the rest of the state machine
+Event::Start(orgize::Element::Link(link)) => {
+    let dest = link.path.to_string();
+    let (proto, stripped_dest): (Option<String>, String) =
+        match dest.split_once(':') {
+            Some((proto, stripped_dest)) => {
+                (Some(proto.to_string()), stripped_dest.to_string())
+            }
+            None => (None, dest.clone()),
+        };
+
+    let last_non_none = match id_crumbs.iter().rev().find_map(|x| x.clone()) {
+        Some(last_non_none) => last_non_none,
+        None => {
+            return Err(InvalidDocError::new_err(format!(
+                "no non-none ID in {}",
+                path
+            )));
+        }
+    };
+
+    let link_list = links.entry(last_non_none.clone()).or_insert(Vec::new());
+    link_list.push(Link {
+        from_file: path.clone().to_string(),
+        from_id: last_non_none.clone(),
+        to: stripped_dest.clone(),
+        to_proto: proto.clone(),
+        text: link.desc.clone().map(String::from),
+    });
+    Ok(())
+}
+// Link parsing:1 ends here
+
+// [[file:../arroyo-native-parser.org::*Cleaning up][Cleaning up:1]]
+                _ => Ok(()),
+            }
+        })
+        .fold_ok(true, |_, _result| true)?;
+
+    return Ok(headings
+        .iter()
+        .map(|heading| {
+            let mut h = heading.clone();
+            h.links = links
+                .get(&h.id.clone().unwrap_or(String::from("")))
+                .cloned();
+            h
+        })
+        .collect());
+}
+// Cleaning up:1 ends here
+
+// [[file:../arroyo-native-parser.org::*=split_quoted_string=][=split_quoted_string=:1]]
+fn split_quoted_string(quoted_str: String) -> Result<Vec<String>, Box<dyn Error>> {
+    let str_as_list = format!("[{}]", &quoted_str);
+    let mut sexp =
+        lexpr::parse::Parser::from_str_custom(&str_as_list, lexpr::parse::Options::elisp());
+    let ret: Vec<String> = sexp
+        .value_iter()
+        .filter_map(|val| {
+            let ret: Option<Vec<String>> = match val {
+                Ok(lexpr::Value::Null) => None,
+                Ok(lexpr::Value::Vector(values)) => Some(
+                    values
+                        .iter()
+                        .map(|inner_val| match inner_val {
+                            lexpr::Value::String(string) => string.to_string(),
+                            lexpr::Value::Symbol(string) => string.to_string(),
+                            others => todo!("{:?}", others),
+                        })
+                        .collect(),
+                ),
+                Ok(lexpr::Value::Symbol(sym)) => Some(vec![sym.to_string()]),
+                Err(_) => todo!(),
+                val => todo!("{:?}", val),
+            };
+            ret
+        })
+        .flatten()
+        .collect();
+
+    Ok(ret)
+}
+// =split_quoted_string=:1 ends here
+
+// [[file:../arroyo-native-parser.org::*Tests][Tests:1]]
+#[cfg(test)]
+mod tests {
+    use std::assert_eq;
+
+    use crate::parse::*;
+
+    #[test]
+    fn test_split_quoted_string() {
+        assert_eq!(
+            split_quoted_string(String::from("")).expect("Could not parse"),
+            Vec::<String>::new()
+        );
+
+        assert_eq!(
+            split_quoted_string(String::from("CCE")).expect("Could not parse"),
+            vec!["CCE"]
+        );
+
+        assert_eq!(
+            split_quoted_string(String::from("\"CCE but with spaces\"")).expect("Could not parse"),
+            vec!["CCE but with spaces"]
+        );
+
+        assert_eq!(
+            split_quoted_string(String::from("\"CCE but with spaces\" \"And 2\""))
+                .expect("Could not parse"),
+            vec!["CCE but with spaces", "And 2"]
+        );
+
+        assert_eq!(
+            split_quoted_string(String::from("\"CCE but with spaces\" CCE"))
+                .expect("Could not parse"),
+            vec!["CCE but with spaces", "CCE"]
+        );
+    }
+
+    // test for tag inheritence and filetags
+    // test for keyword extraction
+    // test for link parsing
+
+    // A fairly complicated doc
+    #[test]
+    fn test_iter_doc_cce() {
+        let doc = parse_document(String::from("/home/rrix/org/cce/cce.org"))
+            .expect("Did not parse a doc");
+
+        assert_eq!(doc.headings.len(), 4, "Expected four headings in doc");
+
+        assert_eq!(
+            doc.headings[0].text, "The Complete Computing Environment",
+            "Title correctly set"
+        );
+
+        assert_eq!(
+            doc.headings[1].text, "[[id:20211219T184243.333209][\"Hey Smell This\"]]",
+            "Title correctly set"
+        );
+
+        assert!(
+            doc.headings[0].links.as_ref().unwrap().len() > 0,
+            "No links!"
+        );
+
+        assert_eq!(doc.keywords[0].keyword, "TITLE", "Title at da top");
+
+        assert_eq!(
+            doc.collect_keywords("ARCOLOGY_KEY".to_string())
+                .expect("Expected keywords..."),
+            vec!["cce/cce".to_string()],
+            "Keyword collection works"
+        );
+    }
+}
+// Tests:1 ends here
--- a/src/types.rs
+++ b/src/types.rs
@ -0,0 +1,185 @@
+// [[file:../arroyo-native-parser.org::*The Parser Document Types][The Parser Document Types:1]]
+use pyo3::exceptions::PyException;
+use pyo3::prelude::*;
+use pyo3::pyclass;
+
+use std::fmt;
+
+pyo3::create_exception!(arroyo_rs, InvalidDocError, PyException);
+// The Parser Document Types:1 ends here
+
+// [[file:../arroyo-native-parser.org::*Document][Document:1]]
+#[derive(Debug, Clone, Default)]
+#[pyclass(dict)]
+pub struct Document {
+    #[pyo3(get)]
+    pub path: String,
+    #[pyo3(get)]
+    pub headings: Vec<Heading>,
+    #[pyo3(get)]
+    pub keywords: Vec<Keyword>,
+}
+// Document:1 ends here
+
+// [[file:../arroyo-native-parser.org::*Document][Document:2]]
+impl fmt::Display for Document {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "OrgDoc(from {} with {} headings and {} keywords)",
+            self.path,
+            self.headings.len(),
+            self.keywords.len()
+        )
+    }
+}
+// Document:2 ends here
+
+// [[file:../arroyo-native-parser.org::*Document][Document:3]]
+#[pymethods]
+impl Document {
+    pub fn __repr__(slf: PyRef<'_, Self>) -> PyResult<String> {
+        Ok(slf.to_string())
+    }
+
+    pub fn __str__(slf: PyRef<'_, Self>) -> PyResult<String> {
+        Self::__repr__(slf)
+    }
+
+    pub fn collect_keywords(&self, keyword: String) -> PyResult<Vec<String>> {
+        let kws: Vec<String> = self
+            .keywords
+            .iter()
+            .filter(|kw| kw.keyword.to_uppercase() == keyword)
+            .map(|kw| kw.value.clone())
+            .collect();
+
+        return Ok(kws);
+    }
+}
+// Document:3 ends here
+
+// [[file:../arroyo-native-parser.org::*Keyword][Keyword:1]]
+#[derive(Debug, Clone)]
+#[pyclass(dict)]
+pub struct Keyword {
+    #[pyo3(get)]
+    pub file: String,
+    #[pyo3(get)]
+    pub keyword: String,
+    #[pyo3(get)]
+    pub value: String,
+}
+
+impl fmt::Display for Keyword {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "Keyword(file={}, {}={})",
+            self.file, self.keyword, self.value
+        )
+    }
+}
+
+#[pymethods]
+impl Keyword {
+    pub fn __repr__(slf: PyRef<'_, Self>) -> PyResult<String> {
+        Ok(slf.to_string())
+    }
+
+    pub fn __str__(slf: PyRef<'_, Self>) -> PyResult<String> {
+        Self::__repr__(slf)
+    }
+}
+// Keyword:1 ends here
+
+// [[file:../arroyo-native-parser.org::*Heading][Heading:1]]
+#[derive(Debug, Clone, Default)]
+#[pyclass(dict)]
+pub struct Heading {
+    // note that some Headlines may not have an ID, but for purpose of
+    // arcology we only care about ones with ID
+    #[pyo3(get)]
+    pub id: Option<String>,
+    #[pyo3(get)]
+    pub level: usize,
+    #[pyo3(get)]
+    pub text: String,
+    #[pyo3(get)]
+    pub tags: Option<Vec<String>>,
+    #[pyo3(get)]
+    pub refs: Option<Vec<String>>,
+    #[pyo3(get)]
+    pub aliases: Option<Vec<String>>,
+
+    #[pyo3(get)]
+    pub links: Option<Vec<Link>>,
+}
+
+impl fmt::Display for Heading {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "Heading(id={}, title={}, {} tags, {} refs, {} aliases, {} links)",
+            self.id.clone().unwrap_or("None".to_owned()),
+            self.text,
+            self.tags.clone().unwrap_or(vec![]).len(),
+            self.refs.clone().unwrap_or(vec![]).len(),
+            self.aliases.clone().unwrap_or(vec![]).len(),
+            self.links.clone().unwrap_or(vec![]).len(),
+        )
+    }
+}
+
+#[pymethods]
+impl Heading {
+    pub fn __repr__(slf: PyRef<'_, Self>) -> PyResult<String> {
+        Ok(slf.to_string())
+    }
+
+    pub fn __str__(slf: PyRef<'_, Self>) -> PyResult<String> {
+        Self::__repr__(slf)
+    }
+}
+// Heading:1 ends here
+
+// [[file:../arroyo-native-parser.org::*Link][Link:1]]
+#[derive(Debug, Clone, Default)]
+#[pyclass(dict)]
+pub struct Link {
+    #[pyo3(get)]
+    pub from_file: String,
+    #[pyo3(get)]
+    pub from_id: String,
+    #[pyo3(get)]
+    pub to: String,
+    #[pyo3(get)]
+    pub to_proto: Option<String>,
+    #[pyo3(get)]
+    pub text: Option<String>,
+}
+
+impl fmt::Display for Link {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "Link({}#{} -> [[{}][{}]])",
+            self.from_file,
+            self.from_id.clone(),
+            self.to,
+            self.text.clone().unwrap_or("".to_owned())
+        )
+    }
+}
+
+#[pymethods]
+impl Link {
+    pub fn __repr__(slf: PyRef<'_, Self>) -> PyResult<String> {
+        Ok(slf.to_string())
+    }
+
+    pub fn __str__(slf: PyRef<'_, Self>) -> PyResult<String> {
+        Self::__repr__(slf)
+    }
+}
+// Link:1 ends here
--- a/test/test_arroyo_rs.py
+++ b/test/test_arroyo_rs.py
@ -0,0 +1,44 @@
+# [[file:../arroyo-native-parser.org::*Tests][Tests:2]]
+import arroyo.arroyo_rs
+import arroyo.models
+
+def test_assert_logic():
+    doc = arroyo.arroyo_rs.parse_file("./arroyo-native-parser.org")
+
+    assert doc.keywords[0].file == "./arroyo-native-parser.org"
+    assert doc.keywords[0].keyword == "TITLE"
+    assert doc.keywords[0].value == "The arroyo_rs Native Org Parser"
+
+    assert doc.headings[0].id == "20231023T115950.248543"
+    assert doc.headings[0].level == 0
+    assert len(doc.headings[0].text) > 0
+    assert len(doc.headings[0].links) > 0
+    assert len(doc.headings[0].aliases) == 2
+    assert doc.headings[0].refs[0] == "https://code.rix.si/rrix/arroyo_rs"
+    assert doc.headings[0].tags == ["Project"]
+    assert doc.headings[-1].tags == ["Project", "Code"]
+
+    assert doc.headings[0].links[0].from_file == "./arroyo-native-parser.org"
+    assert doc.headings[0].links[0].from_id == "20231023T115950.248543"
+    assert doc.headings[0].links[0].to == "id:1fb8fb45-fac5-4449-a347-d55118bb377e"
+    assert doc.headings[0].links[0].to_proto == "id"
+    assert doc.headings[0].links[0].text == "org-mode"
+
+def test_relationships():
+    assert arroyo.arroyo_rs.Heading is not None
+    assert arroyo.arroyo_rs.Document is not None
+    assert arroyo.arroyo_rs.Keyword is not None
+    assert arroyo.arroyo_rs.Link is not None
+
+# def test_sqlmodel_conversion():
+#   native = arroyo.arroyo_rs.parse_file("./arroyo-native-parser.org")
+#   taglinks = arroyo.models.Tag.from_native_doc(native)
+#   headings = arroyo.models.Heading.from_native_doc(native)
+#   keywords = arroyo.models.Keyword.from_native_doc(native)
+#   document = arroyo.models.Document.from_native_doc(native)
+#
+#   assert(len(keywords) == 5)
+#   assert(headings[0].node_id == '20231023T115950.248543')
+#   assert(headings[1].node_id == None)
+#   assert(headings[1].text == "Overview")
+# Tests:2 ends here
Author	SHA1	Message	Date
Ryan Rix	8781e8a1dd	switch arroyo-emacs to shell out to Arcology Generators	2023-12-27 19:32:08 -08:00
Ryan Rix	5ac820fd71	re-jiggle link handling to mark missing ID links as 404	2023-12-27 19:31:41 -08:00
Ryan Rix	80135fea4e	make sure htmlize_file and its options are in the public iface	2023-12-27 18:19:33 -08:00
Ryan Rix	778d5f47f3	if a link is html, always export it	2023-12-27 18:19:12 -08:00
Ryan Rix	62eaf82ecf	switch to flake for direnv	2023-12-27 18:18:48 -08:00
Ryan Rix	71264fe99c	use arcology-django to run arroyo interface commands	2023-12-18 23:05:06 -08:00
Ryan Rix	6b0deb66e5	update to new keywords	2023-12-17 16:06:19 -08:00
Ryan Rix	db09908f89	hoist the drawer state handling + also elide ends of things inside the drawers	2023-12-11 11:45:01 -08:00
Ryan Rix	ae721adc47	refactor link export to be a bit more legible	2023-12-11 09:33:49 -08:00
Ryan Rix	827af0be41	bit of text tweaks	2023-12-11 00:12:26 -08:00
Ryan Rix	571d806b23	customized html exporting	2023-12-11 00:07:25 -08:00
Ryan Rix	4c5e1ac2ce	fix orgize build error woops	2023-12-04 14:11:36 -08:00
Ryan Rix	72f927efe9	don't need watchfiles self-package any more...	2023-12-04 14:11:20 -08:00
Ryan Rix	cab5986076	load the overlay in to <arroyo> default.nix	2023-12-04 14:11:15 -08:00
Ryan Rix	b3294d91bd	delete-trailing-whitespace	2023-11-20 10:56:43 -08:00
Ryan Rix	7d683c3f08	start to think about exporter features	2023-11-20 10:56:13 -08:00
Ryan Rix	d496421ef2	add todo_keywords to arroyo exporter	2023-11-20 10:55:39 -08:00
Ryan Rix	e6622ee303	add tests for collect_keyword	2023-11-15 12:08:02 -08:00
Ryan Rix	ef548f49a6	fix up some formatting	2023-11-15 12:07:54 -08:00
Ryan Rix	aadecb1f5c	collect_keywords oughta return the right data	2023-11-15 12:06:00 -08:00
Ryan Rix	7d0c303d20	make the extension-module stuff optional to only the python	2023-11-15 12:05:01 -08:00
Ryan Rix	1af343ebbc	return values from collect_keywords, not the keyword itself...	2023-11-15 11:08:40 -08:00
Ryan Rix	b295daee35	split protocol out of "to" in Link	2023-11-14 12:53:25 -08:00
Ryan Rix	ccbdca95c1	find_keywords -> collect_keywords	2023-11-14 11:51:53 -08:00
Ryan Rix	0fccdc76bc	clean up the "readme" parts a bit	2023-11-12 23:43:05 -08:00
Ryan Rix	3d1cd6f197	add find_keyword for native Document type	2023-11-12 23:43:05 -08:00
Ryan Rix	ccdbe38450	rip out sqlmodel (for now?)	2023-11-10 14:35:24 -08:00
Ryan Rix	081c5c2270	one more cleanup	2023-11-10 14:19:31 -08:00
Ryan Rix	c25a476658	cleanup Heading type	2023-11-10 14:19:01 -08:00
Ryan Rix	9ac5b5a788	cleanup before i rip a bunch of shit out	2023-11-10 14:18:24 -08:00
Ryan Rix	30c2829a54	make the package `nix run`-able	2023-10-30 18:03:57 -07:00
Ryan Rix	75f5fefa40	fix tangle property for __init__	2023-10-30 18:02:42 -07:00
Ryan Rix	c7e4a86beb	downgrade version to 0.0.1	2023-10-30 18:02:42 -07:00
Ryan Rix	30424b2bce	splat model stuff in	2023-10-30 18:00:24 -07:00
Ryan Rix	6ae76d23df	fix `nix build`	2023-10-30 17:33:56 -07:00
Ryan Rix	b1fd6cd437	reenable tests	2023-10-26 18:05:43 -07:00
Ryan Rix	d307c413ea	update parser entry point	2023-10-26 18:05:43 -07:00
Ryan Rix	47c769f09e	some headings don't have tags and that's okay	2023-10-26 18:05:43 -07:00
Ryan Rix	c1ae1f652e	plumb invalid document errors out to the python interface	2023-10-26 18:03:09 -07:00
Ryan Rix	a95ffc628e	turning on comment links for babel detangle	2023-10-26 15:05:24 -07:00
Ryan Rix	72e1277655	tags are many-to-one instead of many-to-many; easy enough to group by to coalesce them	2023-10-26 15:04:46 -07:00
Ryan Rix	60f8cb5ab0	only store allowlisted keywords	2023-10-26 15:02:41 -07:00
Ryan Rix	c11953a5ae	ID inheritance so that links' "from_id" isn't optional	2023-10-26 15:02:25 -07:00
Ryan Rix	9771bec39e	update gitignore	2023-10-26 00:49:35 -07:00
Ryan Rix	6f32115487	get sqlmodel persistence for the first bits of model working	2023-10-26 00:48:54 -07:00
Ryan Rix	8606460857	wip to plumb native types through	2023-10-25 17:34:30 -07:00
Ryan Rix	8075a6aa2c	don't use poetry for dep management, maturin will use project.dependencies	2023-10-25 17:33:39 -07:00
Ryan Rix	c1a06c6d16	enable detangle comments	2023-10-24 10:35:58 -07:00
Ryan Rix	461bfaa40a	start to integrate sqlmodel	2023-10-23 18:16:20 -07:00
Ryan Rix	3642b12256	fix footnotes for parser	2023-10-23 18:15:25 -07:00
Ryan Rix	072e4dfb0f	track path inside of Document struct	2023-10-23 18:14:31 -07:00
Ryan Rix	f1fb8379d0	clippy fixes	2023-10-23 15:26:51 -07:00
Ryan Rix	73ce89c4cb	clean up deps and test infrastructure	2023-10-23 13:52:25 -07:00
Ryan Rix	5fbbd47d64	splat my arroyo_rs repo in to the arroyo repo	2023-10-23 13:31:33 -07:00