Compare commits

...

2 Commits

Author SHA1 Message Date
Ryan Rix 1c544d18cc ensure css is in the package 2024-02-26 19:50:54 -08:00
Ryan Rix ececb089bc add SigmaJS endpoint 2024-02-26 19:38:51 -08:00
14 changed files with 585 additions and 32 deletions

View File

@ -1147,7 +1147,7 @@ A footer contains the oh-so-important copyright notice and a limited privacy pol
</p>
<p>
View the <a href="/sitemap/">Site Map</a> or the <a href="/tags">Tag Index</a>.
View the <a href="/sitemap">Site Map</a> or the <a href="/tags">Tag Index</a>.
</p>
<p>

View File

@ -74,7 +74,7 @@
</p>
<p>
View the <a href="/sitemap/">Site Map</a> or the <a href="/tags">Tag Index</a>.
View the <a href="/sitemap">Site Map</a> or the <a href="/tags">Tag Index</a>.
</p>
<p>

View File

@ -20,13 +20,15 @@ package-dir = {"" = "."}
[tool.setuptools.package-data]
arcology = [
'settings/sites.json',
'templates/arcology/*',
'templates/*',
'static/arcology/js/*',
'static/arcology/css/*',
'static/arcology/fonts/*',
'templates/arcology/*',
'templates/*',
]
sitemap = [
'static/sitemap/js/*',
'static/sitemap/css/*',
'templates/sitemap/*',
]

View File

@ -32,13 +32,15 @@ package-dir = {"" = "."}
[tool.setuptools.package-data]
arcology = [
'settings/sites.json',
'templates/arcology/*',
'templates/*',
'static/arcology/js/*',
'static/arcology/css/*',
'static/arcology/fonts/*',
'templates/arcology/*',
'templates/*',
]
sitemap = [
'static/sitemap/js/*',
'static/sitemap/css/*',
'templates/sitemap/*',
]

View File

@ -16,12 +16,16 @@ Then there is the Sitemap page which uses [[id:20220711T151820.326251][SigmaJS]]
#+begin_src python :tangle sitemap/views.py :mkdirp yes
import logging
from django.http import HttpResponse, HttpResponseNotFound
import json
import functools
from django.http import HttpResponse, HttpResponseNotFound, JsonResponse
from django.shortcuts import render, get_object_or_404
from django.db.models import Count
from arcology.models import Page, Feed, Site
from roam.models import Link, Tag
from roam.models import Link, Tag, File
from sitemap.models import Node, Edge
# from prometheus_client import Counter, Histogram
@ -167,23 +171,16 @@ def tag_page(request, tag: str):
With some minor CSS rules applied we get a sort of limited "tag cloud" effect where pages with more links to this heading get made larger
#+begin_src css
.tag-index ul.tag-list {
list-style: none;
padding-left: 0;
display: flex;
flex-wrap: wrap;
align-items: center;
justify-content: center;
}
#+end_src
#+begin_src css :tangle sitemap/static/sitemap/css/sitemap.css :mkdirp yes
.tag-index ul.tag-list a {
--size: 1;
font-size: calc(log(var(--size) + 1) * 120%);
}
#sitemap-container {
height: 80em;
filter: saturate(500%);
}
#+end_src
That =request.htmx= branch will make sure we only render the list if the HTMX partial is called by swapping the base HTML template to render only the list:
@ -200,16 +197,338 @@ That =request.htmx= branch will make sure we only render the list if the HTMX pa
That this relies on [[id:20240204T234334.762591][Data Models for Sites, Web Features, and Feeds]] *and* [[id:arcology/django/roam][Arcology Roam Models]] tells me it may be needs to be in a different module, idk... the structure of these projects really does need to be worked on.
This also lifts design and code whole-sale from [[id:arcology/fastapi][Arcology FastAPI]]: [[id:arcology/sitemaps][Navigating the Arcology Site Graph with SigmaJS]].
Nodes come from =Page= objects and are shaped like this:
#+begin_example json
{
"id": "n0",
"label": "A node",
"x": 0,
"y": 0,
"size": 3
}
#+end_example
Edges come from =Link= objects and are shaped like this:
#+begin_example json
{
"id": "e0",
"source": "n0",
"target": "n1"
}
#+end_example
compose together and return in a single dict for JSON rendering shaped like:
#+begin_example json
{
"nodes": [
<<all nodes>>
]
"edges": [
<<all edges>>
]
}
#+end_example
** Making SigmaJS Nodes
#+begin_src python :tangle sitemap/models.py :mkdirp yes
import arcology.models
import roam.models
import functools
import hashlib
def make_loc_hash(page: arcology.models.Page, salt, max_q=700):
key = page.file.path + str(salt)
hash = hashlib.sha224(key.encode("utf-8")).digest()
return int.from_bytes(hash, byteorder="big") % max_q
class Node():
@classmethod
def make_page_dict(cls, page):
@functools.lru_cache(maxsize=5000)
def _make(page, hash):
link_cnt = page.file.outbound_links.count()
backlink_cnt = roam.models.Link.objects.filter(dest_heading__in=page.file.heading_set.all()).count()
return dict(
key=page.route_key,
attributes=dict(
label=page.title,
x=make_loc_hash(page, 1),
y=make_loc_hash(page, 2),
size=min((link_cnt + backlink_cnt) / 2, 20),
color=page.site.link_color,
href=page.to_url(),
)
)
return _make(page, page.file.digest)
@classmethod
def get_sigmajs_nodes(cls):
pages = arcology.models.Page.objects.all()
nodes = [
cls.make_page_dict(page)
for page in pages
]
return nodes
#+end_src
** Making SigmaJS Edges
#+begin_src python :tangle sitemap/models.py :mkdirp yes
class hashabledict(dict):
def __hash__(self):
return hash(tuple(sorted(self.items())))
class Edge():
@classmethod
def get_sigmajs_edges(cls):
q_links = roam.models.Link.objects.all()
links = set()
for link in q_links:
try:
source = link.source_file.page_set.first().route_key
dest = link.dest_heading.path.page_set.first().route_key
links.add(
hashabledict(
key=f"{source}-{dest}",
source=source,
target=dest,
)
)
except roam.models.Heading.DoesNotExist:
pass
return list(links)
#+end_src
** JSON Handler/View
This tries to calculate a consistent cache key cheaply and probably fails.
#+begin_src python :tangle sitemap/views.py :mkdirp yes
import hashlib
@functools.lru_cache(maxsize=20)
def _cached(cache_key, hashes):
print(f"called w/ cache key {cache_key}")
ret = dict(
nodes=Node.get_sigmajs_nodes(),
edges=Edge.get_sigmajs_edges(),
)
print(f"finished call w/ cache key {cache_key}")
return ret
def sitemap_data(request):
pass
hashes = [ file.digest for file in File.objects.order_by('path') ]
cache_key = hashlib.sha224(''.join(hashes).encode("utf-8")).hexdigest()
return JsonResponse(_cached(cache_key, cache_key))
#+end_src
* Sitemap HTML Page
#+begin_src python :tangle sitemap/views.py :mkdirp yes
def sitemap(request):
pass
site = Site.from_request(request)
return render(request, "sitemap/map.html", dict(
site=site
))
#+end_src
** Sitemap Page Template
#+begin_src jinja2 :tangle sitemap/templates/sitemap/map.html
{% extends "arcology/app.html" %}
{% load static %}
{% block h1 %}<h1>{{site.title}}<h2>A Map of the Arcology Sites</h2></h1>{% endblock %}
{% block title %}Arcology Sites Map{% endblock %}
{% block extra_head %}
<link rel="stylesheet" href="{% static 'sitemap/css/sitemap.css' %}"/>
<script src="{% static 'sitemap/js/htmx.js' %}" defer></script>
<script src="{% static 'sitemap/js/graphology.min.js' %}" defer></script>
<script src="{% static 'sitemap/js/graphology-library.min.js' %}" defer></script>
<script src="{% static 'sitemap/js/sigmajs.min.js' %}" defer></script>
{% endblock %}
{% block content %}
<section>
<p>
This is a network graph of all the pages published in my
Arcology network. The color of the node corresponds to its
site, and you can click on any of them to jump to that page or
zoom in to see more of their labels. I will be adding more
features to this sitemap like search and highlighting
"relative" neighbors later on. Feel free to view
<a href="https://engine.arcology.garden/sitemaps">the implementation</a>
within the Arcology pages, as usual.
</p>
<p>
You may also like to see the <a href="/tags/">Tag Index</a>.
</p>
<div id="sitemap-container">
</div>
<script type="application/javascript" src="{% static 'sitemap/js/sitemap.js'%}"></script>
</section>
{% endblock %}
#+end_src
** JavaScript to set up and render the SigmaJS graph
The =SigmaJS= upstream code is loaded in the =<head>=, then this code runs after page load to fetch the graph data and cram it in to the =#sitemap-container= =<div>=.
#+begin_src javascript :noweb-ref sigmaInit :noweb yes :tangle sitemap/static/sitemap/js/sitemap.js
const container = document.getElementById("sitemap-container");
fetch('/sitemap.json')
.then(response => response.json())
.then((data) => {
const graph = new graphology.Graph.from(data);
<<forceAtlas2-setup>>
const renderer = new Sigma(graph, container);
<<hoverReducer>>
<<forceAtlas2-invoke-worker>>
<<clickNode>>
return {renderer, graph}
});
#+end_src
A [[https://graphology.github.io/standard-library/layout-forceatlas2.html][ForceAtlas2]] layout algorithm is applied to the page for 5 seconds to allow the nodes to coalesce in to a stable layout.
#+begin_src javascript :noweb-ref forceAtlas2-setup
var FA2Layout = graphologyLibrary.FA2Layout;
var forceAtlas2 = graphologyLibrary.layoutForceAtlas2;
const sensibleSettings = forceAtlas2.inferSettings(graph);
const layout = new FA2Layout(graph, {
settings: sensibleSettings
});
#+end_src
The layout work is done in a [[https://graphology.github.io/standard-library/layout-forceatlas2.html#webworker][background task]]:
#+begin_src javascript :noweb-ref forceAtlas2-invoke-worker
layout.start();
setTimeout(() => {
layout.stop()
}, 5000);
#+end_src
An event handler is defined for the =clickNode= event to open the URL embedded in the node's attributes added in [[id:20220712T101916.751167][Making SigmaJS Nodes]].
#+begin_src javascript :noweb-ref clickNode
renderer.on("clickNode", ({ node }) => {
var realnode = graph._nodes.get(node);
window.location = realnode.attributes.href;
});
#+end_src
** Focusing on Nodes in the Graph
The =<<hoverReducer>>= code to "highlight" a node by de-emphasizing the nodes it does not neighbor is taken from the [[https://codesandbox.io/s/github/jacomyal/sigma.js/tree/main/examples/use-reducers][SigmaJS examples]] and converted from TypeScript to JavaScript by hand. No I will not set up a NodeJS build chain.
#+begin_src javascript :noweb-ref hoverReducer
state = {};
function setHoveredNode(node) {
if (node) {
state.hoveredNode = node;
state.hoveredNeighbors = new Set(graph.neighbors(node));
} else {
state.hoveredNode = undefined;
state.hoveredNeighbors = undefined;
}
// Refresh rendering:
renderer.refresh();
}
renderer.on("enterNode", ({ node }) => {
setHoveredNode(node);
});
renderer.on("leaveNode", () => {
setHoveredNode(undefined);
});
// Render nodes accordingly to the internal state:
// 1. If a node is selected, it is highlighted
// 2. If there is query, all non-matching nodes are greyed
// 3. If there is a hovered node, all non-neighbor nodes are greyed
renderer.setSetting("nodeReducer", (node, data) => {
const res = { ...data };
if (state.hoveredNeighbors && !state.hoveredNeighbors.has(node) && state.hoveredNode !== node) {
res.label = "";
res.color = "#f6f6f6";
}
if (state.selectedNode === node) {
res.highlighted = true;
} else if (state.suggestions && !state.suggestions.has(node)) {
res.label = "";
res.color = "#f6f6f6";
}
return res;
});
// Render edges accordingly to the internal state:
// 1. If a node is hovered, the edge is hidden if it is not connected to the
// node
// 2. If there is a query, the edge is only visible if it connects two
// suggestions
renderer.setSetting("edgeReducer", (edge, data) => {
const res = { ...data };
if (state.hoveredNode && !graph.hasExtremity(edge, state.hoveredNode)) {
res.hidden = true;
}
if (state.suggestions && (!state.suggestions.has(graph.source(edge)) || !state.suggestions.has(graph.target(edge)))) {
res.hidden = true;
}
return res;
});
#+end_src
* Sitemap Frontend JS using Sigma JS
** What is SigmaJS
:PROPERTIES:
:ID: 20220711T151820.326251
:ROAM_REFS: https://www.sigmajs.org/
:ROAM_ALIASES: SigmaJS
:END:
[[https://www.sigmajs.org/][SigmaJS]] is a JavaScript library for visualizing graphs. My [[id:cce/org-roam][org-roam]] knowledge base is a graph and [[id:arroyo/system-cache][arroyo-db]] contains the public pages and links. As nodes and edges, this makes a fine graph and a lot more "logical" than an XML sitemap or linear walk through the sites.
** Getting SigmaJS
Download and vendor the minified versions:
#+begin_src shell
export GRAPHOLOGY_VER=0.24.1
export SIGMAJS_VER=v2.3.1
curl --location -v -o sitemap/static/sitemap/js/sigmajs.min.js https://github.com/jacomyal/sigma.js/releases/download/$SIGMAJS_VER/sigma.min.js
curl --location -v -o sitemap/static/sitemap/js/graphology.min.js https://github.com/graphology/graphology/releases/download/$GRAPHOLOGY_VER/graphology.min.js
curl --location -v -o sitemap/static/sitemap/js/graphology-library.min.js https://github.com/graphology/graphology/releases/download/$GRAPHOLOGY_VER/graphology-library.min.js
#+end_src
These can now be used by the django =static= template helper.

69
sitemap/models.py Normal file
View File

@ -0,0 +1,69 @@
# [[file:../sitemap.org::*Making SigmaJS Nodes][Making SigmaJS Nodes:1]]
import arcology.models
import roam.models
import functools
import hashlib
def make_loc_hash(page: arcology.models.Page, salt, max_q=700):
key = page.file.path + str(salt)
hash = hashlib.sha224(key.encode("utf-8")).digest()
return int.from_bytes(hash, byteorder="big") % max_q
class Node():
@classmethod
def make_page_dict(cls, page):
@functools.lru_cache(maxsize=5000)
def _make(page, hash):
link_cnt = page.file.outbound_links.count()
backlink_cnt = roam.models.Link.objects.filter(dest_heading__in=page.file.heading_set.all()).count()
return dict(
key=page.route_key,
attributes=dict(
label=page.title,
x=make_loc_hash(page, 1),
y=make_loc_hash(page, 2),
size=min((link_cnt + backlink_cnt) / 2, 20),
color=page.site.link_color,
href=page.to_url(),
)
)
return _make(page, page.file.digest)
@classmethod
def get_sigmajs_nodes(cls):
pages = arcology.models.Page.objects.all()
nodes = [
cls.make_page_dict(page)
for page in pages
]
return nodes
# Making SigmaJS Nodes:1 ends here
# [[file:../sitemap.org::*Making SigmaJS Edges][Making SigmaJS Edges:1]]
class hashabledict(dict):
def __hash__(self):
return hash(tuple(sorted(self.items())))
class Edge():
@classmethod
def get_sigmajs_edges(cls):
q_links = roam.models.Link.objects.all()
links = set()
for link in q_links:
try:
source = link.source_file.page_set.first().route_key
dest = link.dest_heading.path.page_set.first().route_key
links.add(
hashabledict(
key=f"{source}-{dest}",
source=source,
target=dest,
)
)
except roam.models.Heading.DoesNotExist:
pass
return list(links)
# Making SigmaJS Edges:1 ends here

View File

@ -1,6 +1,11 @@
/* [[file:../../../../sitemap.org::*Individual Tag Pages (and list partial)][Individual Tag Pages (and list partial):4]] */
/* [[file:../../../../sitemap.org::*Individual Tag Pages (and list partial)][Individual Tag Pages (and list partial):3]] */
.tag-index ul.tag-list a {
--size: 1;
font-size: calc(log(var(--size) + 1) * 120%);
}
/* Individual Tag Pages (and list partial):4 ends here */
#sitemap-container {
height: 80em;
filter: saturate(500%);
}
/* Individual Tag Pages (and list partial):3 ends here */

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,92 @@
// [[file:../../../../sitemap.org::*JavaScript to set up and render the SigmaJS graph][JavaScript to set up and render the SigmaJS graph:1]]
const container = document.getElementById("sitemap-container");
fetch('/sitemap.json')
.then(response => response.json())
.then((data) => {
const graph = new graphology.Graph.from(data);
var FA2Layout = graphologyLibrary.FA2Layout;
var forceAtlas2 = graphologyLibrary.layoutForceAtlas2;
const sensibleSettings = forceAtlas2.inferSettings(graph);
const layout = new FA2Layout(graph, {
settings: sensibleSettings
});
const renderer = new Sigma(graph, container);
state = {};
function setHoveredNode(node) {
if (node) {
state.hoveredNode = node;
state.hoveredNeighbors = new Set(graph.neighbors(node));
} else {
state.hoveredNode = undefined;
state.hoveredNeighbors = undefined;
}
// Refresh rendering:
renderer.refresh();
}
renderer.on("enterNode", ({ node }) => {
setHoveredNode(node);
});
renderer.on("leaveNode", () => {
setHoveredNode(undefined);
});
// Render nodes accordingly to the internal state:
// 1. If a node is selected, it is highlighted
// 2. If there is query, all non-matching nodes are greyed
// 3. If there is a hovered node, all non-neighbor nodes are greyed
renderer.setSetting("nodeReducer", (node, data) => {
const res = { ...data };
if (state.hoveredNeighbors && !state.hoveredNeighbors.has(node) && state.hoveredNode !== node) {
res.label = "";
res.color = "#f6f6f6";
}
if (state.selectedNode === node) {
res.highlighted = true;
} else if (state.suggestions && !state.suggestions.has(node)) {
res.label = "";
res.color = "#f6f6f6";
}
return res;
});
// Render edges accordingly to the internal state:
// 1. If a node is hovered, the edge is hidden if it is not connected to the
// node
// 2. If there is a query, the edge is only visible if it connects two
// suggestions
renderer.setSetting("edgeReducer", (edge, data) => {
const res = { ...data };
if (state.hoveredNode && !graph.hasExtremity(edge, state.hoveredNode)) {
res.hidden = true;
}
if (state.suggestions && (!state.suggestions.has(graph.source(edge)) || !state.suggestions.has(graph.target(edge)))) {
res.hidden = true;
}
return res;
});
layout.start();
setTimeout(() => {
layout.stop()
}, 5000);
renderer.on("clickNode", ({ node }) => {
var realnode = graph._nodes.get(node);
window.location = realnode.attributes.href;
});
return {renderer, graph}
});
// JavaScript to set up and render the SigmaJS graph:1 ends here

View File

@ -0,0 +1,37 @@
{# [[file:../../../sitemap.org::*Sitemap Page Template][Sitemap Page Template:1]] #}
{% extends "arcology/app.html" %}
{% load static %}
{% block h1 %}<h1>{{site.title}}<h2>A Map of the Arcology Sites</h2></h1>{% endblock %}
{% block title %}Arcology Sites Map{% endblock %}
{% block extra_head %}
<link rel="stylesheet" href="{% static 'sitemap/css/sitemap.css' %}"/>
<script src="{% static 'sitemap/js/htmx.js' %}" defer></script>
<script src="{% static 'sitemap/js/graphology.min.js' %}" defer></script>
<script src="{% static 'sitemap/js/graphology-library.min.js' %}" defer></script>
<script src="{% static 'sitemap/js/sigmajs.min.js' %}" defer></script>
{% endblock %}
{% block content %}
<section>
<p>
This is a network graph of all the pages published in my
Arcology network. The color of the node corresponds to its
site, and you can click on any of them to jump to that page or
zoom in to see more of their labels. I will be adding more
features to this sitemap like search and highlighting
"relative" neighbors later on. Feel free to view
<a href="https://engine.arcology.garden/sitemaps">the implementation</a>
within the Arcology pages, as usual.
</p>
<p>
You may also like to see the <a href="/tags/">Tag Index</a>.
</p>
<div id="sitemap-container">
</div>
<script type="application/javascript" src="{% static 'sitemap/js/sitemap.js'%}"></script>
</section>
{% endblock %}
{# Sitemap Page Template:1 ends here #}

View File

@ -1,7 +1,7 @@
{# [[file:../../../sitemap.org::*Individual Tag Pages (and list partial)][Individual Tag Pages (and list partial):5]] #}
{# [[file:../../../sitemap.org::*Individual Tag Pages (and list partial)][Individual Tag Pages (and list partial):4]] #}
<a href="/tags/{{ tag }}">Show all...</a>
<div class="tag-list">
{% block list %}{% endblock %}
</div>
{# Individual Tag Pages (and list partial):5 ends here #}
{# Individual Tag Pages (and list partial):4 ends here #}

View File

@ -1,11 +1,15 @@
# [[file:../sitemap.org::*Django View Setup][Django View Setup:1]]
import logging
from django.http import HttpResponse, HttpResponseNotFound
import json
import functools
from django.http import HttpResponse, HttpResponseNotFound, JsonResponse
from django.shortcuts import render, get_object_or_404
from django.db.models import Count
from arcology.models import Page, Feed, Site
from roam.models import Link, Tag
from roam.models import Link, Tag, File
from sitemap.models import Node, Edge
# from prometheus_client import Counter, Histogram
@ -48,12 +52,30 @@ def tag_page(request, tag: str):
))
# Individual Tag Pages (and list partial):1 ends here
# [[file:../sitemap.org::*Sitemap JSON][Sitemap JSON:1]]
# [[file:../sitemap.org::*JSON Handler/View][JSON Handler/View:1]]
import hashlib
@functools.lru_cache(maxsize=20)
def _cached(cache_key, hashes):
print(f"called w/ cache key {cache_key}")
ret = dict(
nodes=Node.get_sigmajs_nodes(),
edges=Edge.get_sigmajs_edges(),
)
print(f"finished call w/ cache key {cache_key}")
return ret
def sitemap_data(request):
pass
# Sitemap JSON:1 ends here
hashes = [ file.digest for file in File.objects.order_by('path') ]
cache_key = hashlib.sha224(''.join(hashes).encode("utf-8")).hexdigest()
return JsonResponse(_cached(cache_key, cache_key))
# JSON Handler/View:1 ends here
# [[file:../sitemap.org::*Sitemap HTML Page][Sitemap HTML Page:1]]
def sitemap(request):
pass
site = Site.from_request(request)
return render(request, "sitemap/map.html", dict(
site=site
))
# Sitemap HTML Page:1 ends here