splat. f1rst post. canonical code at https://arcology.garden/koreader-notes
Ryan Rix
commit
18c3bf2632
|
|
@ -0,0 +1,189 @@
|
|||
(local pretty (require :pl.pretty))
|
||||
(local stringx (require :pl.stringx))
|
||||
(local tablex (require :pl.tablex))
|
||||
(local text (require :pl.text))
|
||||
(local file (require :pl.file))
|
||||
(local sha256 (require :hashings.sha256))
|
||||
|
||||
(fn collect-highlights [base-path match-name converter-fn]
|
||||
(let [proc (io.popen (.. "bash -c 'find " base-path " -regextype posix-egrep -regex \"" match-name "\" -type f'"))]
|
||||
(local output [])
|
||||
(each [line (proc:lines)]
|
||||
(table.insert
|
||||
output
|
||||
(converter-fn ((loadfile line)) line)))
|
||||
output))
|
||||
|
||||
(fn init-datastructure [path from-md]
|
||||
(let [props (?. from-md :doc_props)
|
||||
stats (?. from-md :stats)]
|
||||
{ "path" path
|
||||
"authors" (.. (?. props :authors))
|
||||
"title" (?. props :title)
|
||||
|
||||
"series" (?. props :series)
|
||||
"md5" (or (?. stats :md5) (?. from-md :partial_md5_checksum))
|
||||
"highlights" []}))
|
||||
|
||||
(fn sort-by-datetime [first second]
|
||||
(< (. first "datetime")
|
||||
(. second "datetime")))
|
||||
|
||||
(fn sort-by-page-no [first second]
|
||||
(< (. first "page")
|
||||
(. second "page")))
|
||||
|
||||
(fn find-chapter-by-name [highlights name]
|
||||
(tablex.find_if highlights (lambda [v]
|
||||
(= (. v :name) name))))
|
||||
|
||||
(fn process-one-book [metadata file-name]
|
||||
(let [bookmarks (. metadata "bookmarks")]
|
||||
(local output (init-datastructure file-name metadata)) ;;
|
||||
|
||||
(print "processing..." (?. (?. metadata "doc_props") "title"))
|
||||
(var sum (accumulate [total 0 _i1
|
||||
inner-tbl (tablex.sortv bookmarks sort-by-datetime)] ;;
|
||||
(do
|
||||
(let [chapter (or (?. inner-tbl "chapter") "")
|
||||
chapter-idx-maybe
|
||||
(or (find-chapter-by-name output.highlights chapter)
|
||||
(do
|
||||
(table.insert output.highlights {:name chapter})
|
||||
(length output.highlights)))
|
||||
]
|
||||
|
||||
(fn mk-highlight [chapter highlight-tbl]
|
||||
{ "datetime" (?. highlight-tbl "datetime")
|
||||
"chapter" chapter
|
||||
"locs" [(?. highlight-tbl "pos0")
|
||||
(?. highlight-tbl "pos1")]
|
||||
"text" (or (?. highlight-tbl "text")
|
||||
(?. highlight-tbl "notes"))})
|
||||
|
||||
(table.insert
|
||||
(. output.highlights chapter-idx-maybe)
|
||||
(mk-highlight chapter inner-tbl)))
|
||||
(+ total 1))))
|
||||
(print "parsed" sum "highlights")
|
||||
output))
|
||||
|
||||
(local template (. text :Template))
|
||||
|
||||
(local highlight-tmpl (template
|
||||
"** ${text}
|
||||
:PROPERTIES:
|
||||
:ID: ${id}
|
||||
:LOC0: ${loc0}
|
||||
:LOC1: ${loc1}
|
||||
:PAGE: ${page}
|
||||
:END:
|
||||
[${datetime}]
|
||||
"))
|
||||
|
||||
(fn render-one-highlight [hl]
|
||||
(let [locs (. hl :locs)
|
||||
digest (: sha256 :new (.. (. hl :text) (. hl :datetime)))
|
||||
hexdigest (: digest :hexdigest)
|
||||
fields (tablex.update
|
||||
{ :datetime (. hl :datetime)
|
||||
:text (-> (. hl :text)
|
||||
(: :gsub "%$" "$ ")
|
||||
(: :gsub "\n" "¶ ")
|
||||
)
|
||||
:id hexdigest }
|
||||
(if (= (type (. locs 1)) "table")
|
||||
{ :page (or (?. hl :page) (?. (. locs 1) :page) "")
|
||||
:loc0 ""
|
||||
:loc1 ""}
|
||||
{ :page ""
|
||||
:loc0 (or (. locs 1) "")
|
||||
:loc1 (or (. locs 2) "")}))]
|
||||
(: highlight-tmpl :substitute
|
||||
fields)))
|
||||
(local chapter-tmpl (template
|
||||
"* ${chapter}
|
||||
"))
|
||||
|
||||
(fn render-one-chapter [chapter]
|
||||
(let [name (?. chapter :name)
|
||||
copy (tablex.deepcopy chapter)]
|
||||
(tset copy :name nil)
|
||||
(stringx.join "\n"
|
||||
(tablex.insertvalues [(: chapter-tmpl :substitute {:chapter name})]
|
||||
(icollect [_i2 hl (ipairs copy)]
|
||||
(values (render-one-highlight hl)))))))
|
||||
(local book-tmpl (template
|
||||
":PROPERTIES:
|
||||
:ID: koreader-${md5}
|
||||
:END:
|
||||
#+TITLE: Notes from ${title}
|
||||
#+AUTHORS: ${authors}
|
||||
[[file:${path}][${path}]]
|
||||
"))
|
||||
|
||||
(fn render-one-book [book]
|
||||
(let [authors (?. book "authors")
|
||||
title (?. book "title")]
|
||||
(.. (: book-tmpl :substitute book)
|
||||
(stringx.join "\n"
|
||||
(icollect [_i1 chapter-hls (pairs (?. book "highlights"))]
|
||||
(render-one-chapter chapter-hls))))))
|
||||
|
||||
(fn maybe-write-to-file [src-file dest-file render-fn]
|
||||
(let [book-mtime (file.modified_time src-file)
|
||||
notes-mtime (file.modified_time dest-file)]
|
||||
(print "Targeting..." dest-file)
|
||||
(if (or (not notes-mtime) ;;
|
||||
(< notes-mtime book-mtime))
|
||||
(match (io.output dest-file)
|
||||
(nil msg) (print "Could not write file... " msg)
|
||||
f (let [text (render-fn)]
|
||||
(io.write text)
|
||||
(io.close f)
|
||||
(print "Rendered..." (length text))))
|
||||
(print "Skipping ... " dest-file))))
|
||||
|
||||
(fn write-one-book [book]
|
||||
(let [book-path (?. book :path)
|
||||
title (?. book :title)
|
||||
notes-path (.. "/home/rrix/org/highlights/" (: title :gsub "[:/\\ ]" "_") ".org")]
|
||||
(print "Maybe rendering" book-path)
|
||||
(maybe-write-to-file book-path
|
||||
notes-path
|
||||
(lambda []
|
||||
(print "rendering" (accumulate [sum 0 i chap (ipairs (. book :highlights))]
|
||||
(+ sum (length chap))) "highlights to string")
|
||||
(render-one-book book)))))
|
||||
|
||||
(fn write-one-book-from-md [md filename]
|
||||
(let [book (process-one-book md filename)]
|
||||
(write-one-book book)
|
||||
book))
|
||||
|
||||
(fn collect-epub-highlights [base-path]
|
||||
(collect-highlights base-path ".*sdr/metadata.(epub|mobi).lua"
|
||||
write-one-book-from-md))
|
||||
|
||||
(fn collect-pdf-highlights [base-path]
|
||||
(collect-highlights
|
||||
base-path ".*sdr/metadata.pdf.lua"
|
||||
write-one-book-from-md))
|
||||
|
||||
(local lapp (require :pl.lapp))
|
||||
|
||||
(let [default-dir "~/mobile-library/books"
|
||||
args (lapp (stringx.join "\n"
|
||||
["Parse koreader metadata files in to org-mode notes"
|
||||
"-f,--file (optional string) only parse one metadata.*.lua file"
|
||||
(.. "-e,--epubs (optional string) parse epubs from here, otherwise " default-dir)
|
||||
(.. "-p,--pdfs (optional string) parse pdfs from here, otherwise " default-dir)]))
|
||||
file-name (?. args :file)
|
||||
file? (not (not file-name))
|
||||
epub-dir (or (. args :epubs) default-dir)
|
||||
pdf-dir (or (. args :pdfs) default-dir)]
|
||||
(if file?
|
||||
(write-one-book-from-md ((loadfile file-name)) file-name)
|
||||
(do
|
||||
(collect-epub-highlights epub-dir)
|
||||
(collect-pdf-highlights pdf-dir))))
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
{ pkgs, lua }:
|
||||
|
||||
rec {
|
||||
nums = lua.pkgs.buildLuarocksPackage {
|
||||
pname = "nums";
|
||||
version = "20130228-2";
|
||||
|
||||
knownRockspec = (pkgs.fetchurl {
|
||||
url = "https://luarocks.org/manifests/user-none/lua-nums-scm-1.rockspec";
|
||||
sha256 = "sha256-fxfcfiAgGGRhyCQZYYdUPs/WplMWVZH4QEPRlSW53uE=";
|
||||
}).outPath;
|
||||
|
||||
src = pkgs.fetchFromGitHub {
|
||||
repo = "lua-nums";
|
||||
owner = "user-none";
|
||||
rev = "fef161a940aaafdbb8d9c75fe073b8bb43152474";
|
||||
sha256 = "sha256-coI8JHMx+6sikSndfbUIuo1jutHUnM3licI2s7I7fmQ=";
|
||||
};
|
||||
|
||||
disabled = with lua; (lua.pkgs.luaOlder "5.3") || (lua.pkgs.luaAtLeast "5.5");
|
||||
|
||||
meta = {
|
||||
homepage = "https://github.com/user-none/lua-nums";
|
||||
description = "Pure Lua number library providing BigNum and fixed width unsigned integer types";
|
||||
license.fullName = "MIT";
|
||||
};
|
||||
};
|
||||
|
||||
hashings =lua.pkgs.buildLuarocksPackage {
|
||||
pname = "hashings";
|
||||
version = "20130228-2";
|
||||
|
||||
knownRockspec = (pkgs.fetchurl {
|
||||
url = "https://luarocks.org/manifests/user-none/lua-hashings-scm-1.rockspec";
|
||||
sha256 = "sha256-SGx6kYhigTCmJQr/lFW6TARpM3na18M8lzgIDcOiCg0=";
|
||||
}).outPath;
|
||||
|
||||
src = pkgs.fetchFromGitHub {
|
||||
repo = "lua-hashings";
|
||||
owner = "user-none";
|
||||
rev = "89879fe79b6f3dc495c607494126ec9c3912b8e9";
|
||||
sha256 = "sha256-/YagiUKAQKtHicsNE4amkHOJZvBEpDMs0qVjszkYnw4=";
|
||||
};
|
||||
|
||||
disabled = with lua; (lua.pkgs.luaOlder "5.3") || (lua.pkgs.luaAtLeast "5.5");
|
||||
propagatedBuildInputs = [ lua nums ];
|
||||
|
||||
meta = {
|
||||
homepage = "https://github.com/user-none/lua-hashings";
|
||||
description = "Pure Lua cryptographic hash library";
|
||||
license.fullName = "MIT";
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
{ pkgs ? import <nixpkgs> {}, ...}:
|
||||
|
||||
let
|
||||
lua = pkgs.lua5_3;
|
||||
myLuaPkgs = import ./pkgs.nix { inherit pkgs; inherit lua; };
|
||||
myHashings = myLuaPkgs.hashings;
|
||||
myLua = lua.withPackages (luapkgs: with luapkgs; [penlight myHashings]);
|
||||
in
|
||||
pkgs.mkShell {
|
||||
packages = [
|
||||
(pkgs.fennel.override (old: old // { lua = myLua; }))
|
||||
myLua
|
||||
pkgs.findutils
|
||||
# inotify-tools
|
||||
];
|
||||
}
|
||||
Loading…
Reference in New Issue