330 lines
11 KiB
Org Mode
330 lines
11 KiB
Org Mode
#+TITLE: Arcology Sitemap
|
|
#+ROAM_ALIAS: Arcology.Sitemap "SigmaJS Site Map" "ArcologyWeb.Sigma"
|
|
#+ARCOLOGY_KEY: arcology/sitemap_module
|
|
|
|
Navigating the [[file:README.org][Arcology]] is supposed to be a somewhat curated experience. I either send out a direct link to a page, and people can read and explore from there, or they go to one of the index pages like [[file:../the_arcology_garden.org][The Arcology Garden]], [[file:../cce/cce.org][The Complete Computing Environment]], or [[file:../the_lions_rear.org][The Lion's Rear]]. Those index pages are carefully curated, if not to send people on a romp through the respective garden, then to at least not get lost if they go back to that page.
|
|
|
|
That said, I've always had a visual streak, I like being able to see the forest for the trees, as it were. Being able to see the full set of relationships is a useful thing; [[file:../cce/org-roam.org][org-roam]] provides a server process which can provide a relationship graph for a page or my entire [[file:../knowledge_base.org][Knowledge Base]], but I would like to have something similar directly in the [[file:README.org][Arcology]]. That is what this page describes, a sitemap which rather than listing the pages in a heirarchy lists them in a connected graph.
|
|
|
|
For now, you have to navigate it somewhat blindly, the locations are not entirely deterministic as a graph settling algorithm is run after the pages are spit out on the canvas deterministically based on a hash of their title and id.
|
|
|
|
* Generating the Graph =ArcologyWeb.Sigma=
|
|
|
|
The =ArcologyWeb.Sigma= module generates a =Map= of the site, a sitemap. The shape of this map is specifically tuned to the input format of Sigma JS, the map should be able to be fed to the JS frontend with a pass through a JSON serialize/deserialize.
|
|
=network/1= and =network/0= create lists of =ArcologyWeb.Sigma.Node= and =ArcologyWeb.Sigma.Edge= structures and then converts them to a big map.
|
|
|
|
#+begin_src elixir :tangle lib/arcology_web/sigma.ex :noweb yes
|
|
defmodule ArcologyWeb.Sigma do
|
|
@moduledoc "Generates a map in the shape of Sigma JS network"
|
|
|
|
alias ArcologyWeb.Sigma.Node
|
|
alias ArcologyWeb.Sigma.Edge
|
|
|
|
use Memoize
|
|
|
|
def network(), do: network(Arcology.Roam.Link.files())
|
|
defmemo network(links) do
|
|
nodes =
|
|
links
|
|
|> Node.from_links()
|
|
|> Enum.map(&Node.add_hashed_locations(&1))
|
|
|> Enum.map(&Node.add_site_colors(&1))
|
|
|> Enum.map(&Node.to_map(&1))
|
|
|
|
edges =
|
|
links
|
|
|> Edge.from_links()
|
|
|> Enum.map(&Edge.to_map(&1))
|
|
|
|
%{nodes: nodes, edges: edges}
|
|
end
|
|
end
|
|
#+end_src
|
|
|
|
** Node
|
|
|
|
Sigma nodes exist to make it easy to translate a link in to two nodes (which are closest really to [[file:arcology_roam.org][Arcology.Roam.File]]s, than anything they are here). =from_links/1= takes a list and =from_link/3= takes an [[file:arcology_roam.org][Arcology.Roam.Link]], and are described below. This code is, frankly, brutish, and I'll have to rewrite all of this once I better understand how elixir structs work. these =Map.get= calls everywhere are not great, I don't understand why =structs= don't implement =Access=...
|
|
|
|
#+begin_src elixir :tangle lib/arcology_web/sigma/node.ex :noweb yes :mkdirp yes
|
|
defmodule ArcologyWeb.Sigma.Node do
|
|
alias ArcologyWeb.Sigma.Node
|
|
defstruct [:id, :title, :url, :x, :y, :color]
|
|
|
|
import Ecto.Query
|
|
|
|
def to_map(node) do
|
|
%{
|
|
id: Map.get(node, :id),
|
|
title: Map.get(node, :title),
|
|
label: Map.get(node, :title),
|
|
url: Map.get(node, :url),
|
|
x: Map.get(node, :x),
|
|
y: Map.get(node, :y),
|
|
color: Map.get(node, :color),
|
|
}
|
|
end
|
|
|
|
<<node-from_links>>
|
|
<<node-from_link>>
|
|
|
|
<<node-modifiers>>
|
|
end
|
|
#+end_src
|
|
=from_links/1= loads a list of all the [[file:arcology_roam.org][Arcology.Roam.Title]] and [[file:arcology_roam.org][Arcology.Roam.Keyword]] objects. These are sent with the link to =from_file/3=.
|
|
|
|
#+begin_src elixir :noweb-ref node-from_links
|
|
def from_links(links) do
|
|
# load keywords for all these files
|
|
keywords = Arcology.Roam.Keyword.all("ARCOLOGY_KEY")
|
|
titles = Arcology.Roam.Title.all()
|
|
|
|
Enum.map(links, &from_link(&1, keywords, titles))
|
|
|> List.flatten()
|
|
|> Enum.filter(&!is_nil(&1))
|
|
|> MapSet.new()
|
|
|> MapSet.to_list()
|
|
end
|
|
#+end_src
|
|
|
|
#+begin_src elixir :tangle test/arcology_web/sigma/node.ex :mkdirp yes
|
|
defmodule ArcologyWeb.SigmaNodeTests do
|
|
use ExUnit.Case
|
|
|
|
import Ecto.Query
|
|
|
|
test 'every file is in the node graph' do
|
|
node_list =
|
|
Arcology.Roam.Links.files()
|
|
|> ArcologyWeb.Sigma.Node.from_links()
|
|
|
|
file_count = from l: Arcology.Roam.File, select: count(l)
|
|
assert length(node_list) == file_count
|
|
end
|
|
end
|
|
#+end_src
|
|
=from_link/3= assembles =Node= objects. This =url_for_arcology_key= needs to be put in the same place as the code for mapping colors, and the Keyword-based routing code in [[file:page_controller.org][ArcologyWeb.PageController]]... Eventually.
|
|
|
|
#+begin_src elixir :noweb-ref node-from_link
|
|
@doc "return a pair of nodes based on the link. keywords and titles are lists of possible objects"
|
|
def from_link(link, keywords, titles) do
|
|
for direction <- [:source, :dest] do
|
|
key =
|
|
Enum.find(keywords, fn kw ->
|
|
!is_binary(link) && kw.file == Map.get(link, direction)
|
|
end)
|
|
|> Arcology.Roam.Keyword.get_value()
|
|
|
|
title =
|
|
Enum.filter(titles, fn title ->
|
|
title.file == Map.get(link, direction)
|
|
end)
|
|
|> Arcology.Roam.Title.to_list()
|
|
|> Enum.at(0)
|
|
|
|
cond do
|
|
key == nil ->
|
|
nil
|
|
|
|
title == nil ->
|
|
%Node{
|
|
id: key,
|
|
title: '(Untitled)',
|
|
url: Arcology.LinkRouter.Local.url_for_arcology_key(key)
|
|
}
|
|
|
|
true ->
|
|
%Node{
|
|
id: key,
|
|
title: title,
|
|
url: Arcology.LinkRouter.Local.url_for_arcology_key(key)
|
|
}
|
|
end
|
|
end
|
|
end
|
|
|
|
#+end_src
|
|
=add_hashed_locations/1= and =add_site_colors/1= take =Node= objects and add somewhat-static locations, and then color the node based on the site in the =ARCOLOGY_KEY=.
|
|
|
|
#+begin_src elixir :noweb-ref node-modifiers
|
|
@doc """
|
|
add hashed coordinates to node.
|
|
|
|
x is the sum of the bytes of the md5 of the id mod 400
|
|
y is the sum of the bytes of the md5 of the title mod 400
|
|
"""
|
|
def add_hashed_locations(node) do
|
|
accumulator = fn last, acc ->
|
|
rem(last + acc, 400)
|
|
end
|
|
|
|
hasherizer = fn text ->
|
|
:crypto.hash(:md5, text)
|
|
|> :binary.bin_to_list()
|
|
|> Enum.reduce(accumulator)
|
|
end
|
|
|
|
%Node{node | x: hasherizer.(Map.get(node, :id)), y: hasherizer.(Map.get(node, :title))}
|
|
end
|
|
|
|
def add_site_colors(node) do
|
|
node_id = Map.get(node, :id)
|
|
|
|
site = Arcology.Page.split_route(node_id) |> Keyword.get(:site)
|
|
|
|
%Node{node | color: Arcology.KeyMaps.site_key_to_color(site)}
|
|
end
|
|
#+end_src
|
|
|
|
** Edge
|
|
|
|
#+begin_src elixir :tangle lib/arcology_web/sigma/edge.ex :noweb yes :mkdirp yes
|
|
defmodule ArcologyWeb.Sigma.Edge do
|
|
alias ArcologyWeb.Sigma.Edge
|
|
defstruct [:id, :source, :target, color: "#BAAD9B", type: "curvedArrow"]
|
|
|
|
import Ecto.Query
|
|
|
|
def from_links(links) do
|
|
# load keywords and titles for all these files only once
|
|
keywords = Arcology.Roam.Keyword.all("ARCOLOGY_KEY")
|
|
titles = Arcology.Roam.Title.all()
|
|
|
|
Enum.map(links, &from_link(&1, keywords, titles))
|
|
|> List.flatten()
|
|
|> Enum.filter(&(&1 != nil))
|
|
|> MapSet.new()
|
|
|> MapSet.to_list()
|
|
end
|
|
|
|
@doc "return a pair of nodes based on the link. keywords and titles are lists of possible objects"
|
|
def from_link(link, keywords, titles) do
|
|
[from_key, to_key] =
|
|
for direction <- [:source, :dest] do
|
|
Enum.find(keywords, fn kw ->
|
|
!is_binary(link) && kw.file == Map.get(link, direction)
|
|
end)
|
|
|> Arcology.Roam.Keyword.get_value()
|
|
end
|
|
|
|
if from_key && to_key do
|
|
%Edge{
|
|
id: from_key <> to_key,
|
|
source: from_key,
|
|
target: to_key
|
|
}
|
|
else
|
|
nil
|
|
end
|
|
end
|
|
|
|
def to_map(edge) do
|
|
%{
|
|
id: Map.get(edge, :id),
|
|
source: Map.get(edge, :source),
|
|
target: Map.get(edge, :target),
|
|
color: Map.get(edge, :color),
|
|
type: Map.get(edge, :type)
|
|
}
|
|
end
|
|
end
|
|
#+end_src
|
|
|
|
* =ArcologyWeb.SitemapController= renders a graphic web of pages
|
|
|
|
There are two actions here, =sigma_network= is called in the JavaScript below, embedded in the page template served by the controller. Nothing special. It loads the text from
|
|
|
|
#+begin_src elixir :noweb yes :tangle lib/arcology_web/controllers/sitemap_controller.ex
|
|
defmodule ArcologyWeb.SitemapController do
|
|
use ArcologyWeb, :controller
|
|
|
|
def sitemap(conn, _params) do
|
|
keywords = Arcology.Roam.Keyword.get("ARCOLOGY_KEY", "arcology/sitemap_template")
|
|
|
|
page =
|
|
keywords
|
|
|> Enum.at(0)
|
|
|> Map.get(:f)
|
|
|> Arcology.Roam.File.preloads()
|
|
|> Arcology.Page.from_file()
|
|
|> Arcology.Page.with_localized_html()
|
|
|
|
conn
|
|
|> render(:sitemap, %{
|
|
site_title: "Arcology",
|
|
page_title: "Sitemap",
|
|
html: page.html
|
|
})
|
|
end
|
|
|
|
def sigma_network(conn, _params) do
|
|
conn
|
|
|> json(ArcologyWeb.Sigma.network())
|
|
end
|
|
end
|
|
#+end_src
|
|
|
|
* JavaScript for rendering the Sitemap
|
|
|
|
This JavaScript is mostly cobbled together from examples provided by [[http://sigmajs.org/][Sigma JS]]; on page load, this code will try to fetch the generated graph from the =/sigma_network= endpoint, through to =ArcologyWeb.Sigma=, and then run a "settling" function for five seconds to make the network legible.
|
|
|
|
#+begin_src javascript :tangle assets/js/s1gma.js
|
|
import * as sigma from 'sigma';
|
|
import 'sigma/build/plugins/sigma.layout.forceAtlas2.min';
|
|
import 'sigma/build/plugins/sigma.plugins.relativeSize.min';
|
|
import 'sigma/build/plugins/sigma.renderers.customEdgeShapes.min';
|
|
|
|
function process(data) {
|
|
var s = new sigma('sigma-network');
|
|
s.settings('labelThreshold', 3);
|
|
s.settings('minArrowSize', 10);
|
|
s.graph.read(data);
|
|
console.log("Starting Force");
|
|
s.startForceAtlas2();
|
|
|
|
s.bind('clickNode', function(data) {
|
|
window.location = data.data.node.url;
|
|
});
|
|
|
|
setTimeout(function() {
|
|
s.stopForceAtlas2();
|
|
sigma.plugins.relativeSize(s, 100);
|
|
s.refresh();
|
|
console.log("Finished Force");
|
|
}, 4000);
|
|
|
|
window.net = s;
|
|
};
|
|
|
|
fetch('/sigma_network')
|
|
.then(res => res.json())
|
|
.then(res => process(res))
|
|
.catch(err => console.error(err));
|
|
#+end_src
|
|
|
|
* Page Template
|
|
|
|
Simple, lol. Just a big ol' div and a reference to that JavaScript.
|
|
|
|
#+begin_src html :tangle lib/arcology_web/templates/sitemap/sitemap.html.eex :mkdirp yes
|
|
<script src="/js/s1gma.js" type="text/javascript"></script>
|
|
|
|
<div>
|
|
<%= raw assigns[:html] %>
|
|
</div>
|
|
|
|
<div style="height: 55em; max-height: 55em;" id="sigma-network"></div>
|
|
#+end_src
|
|
|
|
* =ArcologyWeb.SitemapView=
|
|
|
|
#+begin_src elixir :tangle lib/arcology_web/views/sitemap_view.ex
|
|
defmodule ArcologyWeb.SitemapView do
|
|
use ArcologyWeb, :view
|
|
end
|
|
#+end_src
|
|
|