From 48681b00c561bf147137fac332be53d569c6d377 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Wed, 22 Jul 2026 11:11:03 -0400 Subject: [PATCH] vault-cli: Librarian tooling for the R/J/F/I vault Adds packages/vault-cli (the `vault` CLI: search/show/tree, clobber-safe create/move/rename across both /vault and /data + manifest regen, sync-data, report, analyze) and wires it onto $PATH via modules/hosts/core/vault.nix for server + vault-client hosts. Scripts self-locate siblings via realpath so they run from the Nix store. Includes the deterministic $0 report generator, the topology-manifest generator, and the shared Librarian prompt. --- package.nix | 46 ++++++ vault | 140 +++++++++++++++++++ vault-librarian-prompt.md | 52 +++++++ vault-librarian.py | 222 +++++++++++++++++++++++++++++ vault-topology.py | 71 ++++++++++ vault_lib.py | 285 ++++++++++++++++++++++++++++++++++++++ vault_ops.py | 223 +++++++++++++++++++++++++++++ 7 files changed, 1039 insertions(+) create mode 100644 package.nix create mode 100755 vault create mode 100644 vault-librarian-prompt.md create mode 100644 vault-librarian.py create mode 100644 vault-topology.py create mode 100644 vault_lib.py create mode 100644 vault_ops.py diff --git a/package.nix b/package.nix new file mode 100644 index 0000000..bd334b3 --- /dev/null +++ b/package.nix @@ -0,0 +1,46 @@ +# packages/vault-cli/package.nix — the `vault` CLI + Librarian tooling derivation. +# +# Bundles the dispatcher (`vault`), its write-ops + shared library, the deterministic +# report generator, the topology-manifest generator, and the shared Librarian prompt. +# Exposes `vault` on $PATH. All scripts self-locate their siblings via realpath(__file__), +# so they run identically from the Nix store, the repo, or a dev checkout. python3 is baked +# into the wrapper — the host does not need its own python3. +{ stdenv, python3, lib }: + +stdenv.mkDerivation { + pname = "vault-cli"; + version = "0.1.0"; + src = ./.; + + dontBuild = true; + dontConfigure = true; + + buildInputs = [ python3 ]; + + installPhase = '' + runHook preInstall + + libexec="$out/libexec/vault-cli" + mkdir -p "$libexec" "$out/bin" + + cp vault vault_lib.py vault_ops.py vault-librarian.py vault-topology.py \ + vault-librarian-prompt.md "$libexec/" + chmod +x "$libexec/vault" + + # Wrapper: always invoke our pinned python3 on the staged dispatcher. + makeWrapperArgs="" + printf '#!%s/bin/python3\n' "${python3}" > "$out/bin/vault" + printf 'import os, sys\n' >> "$out/bin/vault" + printf 'os.execv("%s/bin/python3", ["%s/bin/python3", "%s/vault"] + sys.argv[1:])\n' \ + "${python3}" "${python3}" "$libexec" >> "$out/bin/vault" + chmod +x "$out/bin/vault" + + runHook postInstall + ''; + + meta = with lib; { + description = "vault CLI + Librarian tooling for the R/J/F/I Obsidian vault"; + mainProgram = "vault"; + platforms = platforms.linux; + }; +} diff --git a/vault b/vault new file mode 100755 index 0000000..eed850a --- /dev/null +++ b/vault @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +""" +vault — CLI for Ryan's R/J/F/I Obsidian vault. + +Clobber-safe. Dry-run by default; pass --execute to act. +Operates on BOTH trees (/vault map + /data territory) and the topology manifest. + +Subcommands: + vault tree [SYS] print topology (optionally one system) + vault search find IDs/notes by code, title, or content + vault show show an ID's note + its /data home + blob count + vault create "" create ID note (folder+folder-note) in /vault AND /data + vault move <oldID> <newID> renumber: rename note+folder, rewrite backlinks, move /data dir + vault rename <ID> "<title>" retitle in place, rewrite backlinks + vault sync-data reconcile /data skeleton to match manifest (create missing dirs) + vault manifest regenerate the topology manifest note + vault report [--pretty] deterministic Librarian report (delegates to vault-librarian.py) + vault analyze spawn a Hermes Librarian pass (ad-hoc; uses the cron prompt) +""" +import os, re, sys, subprocess + +# Resolve sibling modules whether running from /var/lib/hermes, the repo, or the Nix store. +_SELF_DIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, _SELF_DIR) +import vault_lib as vl + +HERMES = "/run/current-system/sw/bin/hermes" +LIBRARIAN = os.path.join(_SELF_DIR, "vault-librarian.py") +PROMPT = os.path.join(_SELF_DIR, "vault-librarian-prompt.md") + +WIKILINK_RE = re.compile(r'(!?\[\[)([^\]#|]+)((?:#[^\]|]+)?(?:\|[^\]]+)?)(\]\])') + + +def die(msg, code=1): + print("ERROR: " + msg, file=sys.stderr) + sys.exit(code) + + +def executing(argv): + return "--execute" in argv + + +def sys_root(sys_letter): + """Return abs /vault path of a system root, e.g. 'R' -> /vault/R Ryan.""" + s = vl.walk_structure(vl.VAULT, include_meta=True)["systems"] + if sys_letter not in s: + die(f"unknown system '{sys_letter}' (have {sorted(s)})") + return os.path.join(vl.VAULT, s[sys_letter]) + + +def find_id_folder(root, id_code): + """Return abs folder path for an ID code under root, or None.""" + for dp, dns, fns in os.walk(root): + dns[:] = [d for d in dns if d not in vl.SKIP_DIRS] + if vl.id_code(os.path.basename(dp)) == id_code: + return dp + return None + + +# ---------- read-only commands ---------- + +def cmd_tree(argv): + s = vl.walk_structure(vl.VAULT, include_meta=True) + want = argv[0] if argv else None + for code in sorted(s["cats"]): + if want and not code.startswith(want): + continue + n = sum(1 for i in s["ids"] if i.startswith(code + ".")) + print(f" {code:8} {os.path.basename(s['cats'][code])} ({n} IDs)") + + +def cmd_search(argv): + if not argv: + die("usage: vault search <query>") + q = " ".join(argv).lower() + hits = [] + for p in vl.all_notes(vl.VAULT): + rel = os.path.relpath(p, vl.VAULT) + stem = os.path.splitext(os.path.basename(p))[0] + if q in stem.lower() or q in rel.lower(): + hits.append((rel, "name")) + continue + try: + if q in open(p, encoding="utf-8").read().lower(): + hits.append((rel, "content")) + except Exception: + pass + for rel, how in hits[:50]: + print(f" [{how}] {rel}") + print(f"\n{len(hits)} match(es).") + + +def cmd_show(argv): + if not argv: + die("usage: vault show <ID>") + code = argv[0] + root = sys_root(code[0]) + folder = find_id_folder(root, code) + if not folder: + die(f"ID {code} not found in /vault") + print("VAULT folder: " + os.path.relpath(folder, vl.VAULT)) + note = os.path.join(folder, os.path.basename(folder) + ".md") + print("Folder-note: " + ("present" if os.path.exists(note) else "MISSING")) + # /data home + drel = os.path.relpath(folder, vl.VAULT) + dpath = os.path.join(vl.DATA, drel) + if os.path.isdir(dpath): + nblobs = sum(len(fs) for _, _, fs in os.walk(dpath)) + print(f"/data home: {drel} ({nblobs} blobs)") + else: + print(f"/data home: MISSING ({drel})") + + +def main(): + if len(sys.argv) < 2: + print(__doc__) + sys.exit(0) + cmd, argv = sys.argv[1], sys.argv[2:] + dispatch = { + "tree": cmd_tree, + "search": cmd_search, + "show": cmd_show, + } + # write-commands live in vault_ops (imported lazily to keep this file readable) + if cmd in dispatch: + dispatch[cmd](argv) + elif cmd in ("create", "move", "rename", "sync-data", "manifest"): + import vault_ops + vault_ops.run(cmd, argv, executing(sys.argv)) + elif cmd == "report": + sys.exit(subprocess.run([sys.executable, LIBRARIAN] + argv).returncode) + elif cmd == "analyze": + prompt = open(PROMPT, encoding="utf-8").read() + subprocess.run([HERMES, "-z", prompt]) + else: + die(f"unknown command '{cmd}'") + + +if __name__ == "__main__": + main() diff --git a/vault-librarian-prompt.md b/vault-librarian-prompt.md new file mode 100644 index 0000000..7340164 --- /dev/null +++ b/vault-librarian-prompt.md @@ -0,0 +1,52 @@ +You are the **Vault Librarian** for Ryan's R/J/F/I Johnny Decimal Obsidian vault. +Your job: keep the vault coherent, accurate, up to date, and ontologically sound. + +## Context you must load first +1. Load skills: `vault-librarian`, `ryan-vault-rjfi-migration`, `obsidian-vault-navigation`, `johnny-decimal`. +2. The vault (JDex/map) lives at `/vault`; blob territory at `/data`. Notes are the map, + files are the territory, joined by ID number — NEVER copy notes into /data. +3. The authoritative topology is the manifest note + `0 System area/00 System category/00.05 Vault topology (machine-readable)`. +4. All structural operations go through the `vault` CLI at /var/lib/hermes/vault + (clobber-safe, dry-run by default, --execute to act, operates on BOTH trees + manifest). + +## Step 1 — get the deterministic report ($0, no judgment) +Run: python3 /var/lib/hermes/vault-librarian.py --pretty +This gives you: broken wikilinks (introduced vs pre-existing), cross-tree topology diff, +JD structural violations (categories that should be IDs, missing .00/folder-notes, +extend-notation `NN+` leaves that need real IDs, >2-level nesting), to-do harvest, and +category-health signals. Read it fully before deciding anything. + +## Step 2 — AUTO-FIX (silent, deterministic, reversible) +Only do these when the correct action is UNAMBIGUOUS. Use the `vault` CLI, then git-commit. +- Scaffold missing /data skeleton dirs so every vault ID has a blob home (`vault sync-data --execute`). +- Fix a broken wikilink when EXACTLY ONE existing note obviously matches the stale target. +- Create a missing `.00` management note or missing folder-note (empty is fine). +- Check off a to-do checkbox that is verifiably complete (linked artifact exists / the note + itself states it's done). When in doubt, do NOT check it — flag instead. +Commit auto-fixes: cd /vault && git add -A && git commit -m "Librarian: <summary>". + +## Step 3 — FLAG (post to this Matrix room, do NOT act) +Anything ambiguous or structural where the right answer needs Ryan's judgment: +- An ID mis-placed under the wrong category (e.g. journalling under an Identity category), + or overlaps/clashes between IDs. +- Cross-tree orphans (a /data folder with no note, or a note-ID whose /data home you did + not create because the placement is unclear). +- `NN+` extend-notation recipes/items that should get real incremented IDs — propose the IDs. +- Ambiguous broken links (target renamed but multiple candidates, or target never existed). +- To-dos that look done but you can't verify, or stale open to-dos worth surfacing. +- Any single-home violation, sprawl nearing the 10-category / 10-ID limits, or note that + feels ontologically mis-filed. + +NOTE: the `0 System area/09 Archive of pre-RJFI tree` subtree is a SANCTIONED garbage dump +(Ryan's ruling). NEVER flag, fix, or scan inside it — its broken links, stray files, and +odd structure are all intentional/allowed. The tooling already excludes it. + +## Output contract (what you post to the room) +Lead with substance. Structure: +- **Auto-fixed (N):** one terse line each, past tense. Skip this section if none. +- **Flag for you (N):** numbered; each = the issue + your recommended resolution + the exact + `vault` command you'd run if approved. Keep each to ~2 lines. +- If nothing to fix AND nothing to flag: a single line "Vault clean — N notes, no issues." +Be NON-VERBOSE. No preamble, no verification-status chatter. Ryan reads this on mobile +(SchildiNext) — use bold labels + short lines, NEVER markdown tables. diff --git a/vault-librarian.py b/vault-librarian.py new file mode 100644 index 0000000..c4bdea0 --- /dev/null +++ b/vault-librarian.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +""" +vault-librarian.py — deterministic ($0) Librarian report for the R/J/F/I vault. + +Emits every mechanical fact the Librarian judgment pass needs, so the model only +reads a small report instead of the whole vault. NO model calls here. + +Checks: + 1. Broken wikilinks (classified: introduced-guessable vs pre-existing/unresolvable) + 2. Cross-tree topology diff (/vault vs /data; _unsorted exempt) + 3. JD structural violations: + - a category folder that duplicates an existing ID number (the "09 should be 00.09" class) + - missing .00 management note per category + - missing folder-note (Folder Notes plugin) + - extend-notation `NN+ ...` leaves that never got a real ID + - notes nested deeper than the ID level + 4. To-do harvest (- [ ] ...) with file + context + 5. Category health (nearing 10-ID limit; duplicate titles across systems = single-home risk) + +Usage: + python3 vault-librarian.py # JSON + python3 vault-librarian.py --pretty # human-readable markdown +""" +import os, re, sys, json +from collections import defaultdict +sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) +import vault_lib as vl + +TODO_RE = re.compile(r'^\s*[-*]\s+\[ \]\s+(.*)$') +DONE_RE = re.compile(r'^\s*[-*]\s+\[[xX]\]\s+(.*)$') + + +def check_broken_links(): + broken = vl.find_broken_links(vl.VAULT) + guessable, unresolvable = [], [] + for b in broken: + (guessable if b["guess"] else unresolvable).append(b) + return {"guessable": guessable, "unresolvable": unresolvable, + "count": len(broken)} + + +def check_topology(): + return vl.topology_diff() + + +def check_structure(): + out = {"missing_dot00": [], + "missing_folder_note": [], "extend_notation": [], "too_deep": []} + s = vl.walk_structure(vl.VAULT, include_meta=True) + id_nums = set(s["ids"].keys()) + # NOTE: the `09 Archive of pre-RJFI tree` subtree is a deliberate garbage dump + # (Ryan's ruling 2026-07-22) — it is EXCLUDED from all structural checks. Anything + # may accumulate there; we never flag or fix inside it. + for dp, dns, fns in os.walk(vl.VAULT): + dns[:] = [d for d in dns if d not in vl.SKIP_DIRS] + rel = os.path.relpath(dp, vl.VAULT) + if vl.is_archived(rel): + dns[:] = [] + continue + # extend-notation leaves + for fn in fns: + if vl.EXTEND_RE.match(fn): + out["extend_notation"].append(os.path.relpath(os.path.join(dp, fn), vl.VAULT)) + # (b) missing .00 per category + (c) missing folder-note per structural folder + for code, rel in s["cats"].items(): + catf = os.path.join(vl.VAULT, rel) + has_dot00 = any(vl.id_code(d) == code + ".00" + for d in os.listdir(catf) if os.path.isdir(os.path.join(catf, d))) + # a .00 may also be a bare note; check for any NN.00 leaf + if not has_dot00: + out["missing_dot00"].append(code) + # folder-notes: every area/cat/id folder should contain <name>.md + for level in ("areas", "cats", "ids"): + for code, rel in s[level].items(): + f = os.path.join(vl.VAULT, rel) + note = os.path.join(f, os.path.basename(f) + ".md") + if not os.path.exists(note): + out["missing_folder_note"].append(rel) + return out + + +def _todo_bucket(rel, text): + """Classify a to-do as 'long_term' or 'pressing'. + long_term: lives in a spec/design/wishlist doc (big standing checklists we don't nag about). + pressing: journal entries, *Tasks*/*Todo* notes, or near-term signal (a name/date/question).""" + low = rel.lower() + fname = os.path.basename(rel).lower() + # spec/design/wishlist docs -> long term standing lists + long_markers = ("spec", "design", "roadmap", "wishlist", "agora", "about the vault", + "the r-j-f-i system") + if any(m in low for m in long_markers): + return "long_term" + # journal & explicit task notes -> pressing + if "journal" in low or "11.84" in low or "tasks for" in fname or "todo" in fname or "to-do" in fname: + return "pressing" + # near-term signal in the text + t = text.lower() + if t.startswith("respond to") or "awaiting" in t or "?" in text or re.search(r'\b20\d\d-\d\d-\d\d\b', text): + return "pressing" + return "long_term" + + +def check_todos(): + pressing, long_term = [], [] + for p in vl.all_notes(vl.VAULT, include_archive=False): + rel = os.path.relpath(p, vl.VAULT) + try: + lines = open(p, encoding="utf-8").read().splitlines() + except Exception: + continue + for i, line in enumerate(lines): + m = TODO_RE.match(line) + if m: + text = m.group(1).strip() + item = {"file": rel, "line": i + 1, "text": text[:140]} + (pressing if _todo_bucket(rel, text) == "pressing" else long_term).append(item) + return {"pressing": pressing, "long_term": long_term, + "count": len(pressing) + len(long_term)} + + +def check_category_health(): + s = vl.walk_structure(vl.VAULT, include_meta=True) + near_limit = [] + per_cat = defaultdict(int) + for idc in s["ids"]: + per_cat[idc[:idc.rindex(".")]] += 1 + for cat, n in per_cat.items(): + if n >= 9: + near_limit.append({"category": cat, "ids": n}) + # duplicate ID-note titles across systems (single-home smell) + titlemap = defaultdict(list) + for code, name in s["ids"].items(): + title = name.split(" ", 1)[1] if " " in name else name + titlemap[title].append(code) + dup_titles = {t: codes for t, codes in titlemap.items() if len(codes) > 1} + return {"near_id_limit": near_limit, "duplicate_titles": dup_titles} + + +def build_report(): + return { + "counts": vl.build_manifest()["counts"], + "broken_links": check_broken_links(), + "topology": check_topology(), + "structure": check_structure(), + "todos": check_todos(), + "category_health": check_category_health(), + } + + +def pretty(r): + L = [] + c = r["counts"] + L.append(f"# Vault Librarian report") + L.append(f"{c['systems']} systems · {c['areas']} areas · {c['categories']} categories · {c['ids']} IDs\n") + + bl = r["broken_links"] + L.append(f"## Broken wikilinks: {bl['count']}") + if bl["guessable"]: + L.append(f"**Likely fixable ({len(bl['guessable'])})** — target renamed, single candidate:") + for b in bl["guessable"][:30]: + L.append(f" - `{b['file']}`: [[{b['target']}]] → guess [[{b['guess']}]]") + if bl["unresolvable"]: + L.append(f"**Unresolvable ({len(bl['unresolvable'])})** — target never existed / dropped stub:") + for b in bl["unresolvable"][:30]: + L.append(f" - `{b['file']}`: [[{b['target']}]]") + L.append("") + + t = r["topology"] + L.append("## Cross-tree topology (/vault vs /data)") + for level in ("systems", "areas", "cats", "ids"): + md = t[level]["missing_in_data"]; od = t[level]["orphan_in_data"] + if md: + L.append(f" - **{level} missing /data home ({len(md)}):** {', '.join(md[:20])}{' …' if len(md)>20 else ''}") + if od: + L.append(f" - **{level} orphan in /data ({len(od)}):** {', '.join(od[:20])}") + L.append("") + + st = r["structure"] + L.append("## JD structural") + if st["extend_notation"]: + L.append(f" - **extend-notation `NN+` needing real IDs ({len(st['extend_notation'])}):**") + for e in st["extend_notation"]: + L.append(f" {e}") + if st["missing_dot00"]: + L.append(f" - **categories missing a .00 note ({len(st['missing_dot00'])}):** {', '.join(sorted(st['missing_dot00'])[:25])}") + if st["missing_folder_note"]: + L.append(f" - **folders missing a folder-note ({len(st['missing_folder_note'])}):** {len(st['missing_folder_note'])} (Folder Notes plugin)") + L.append("") + + td = r["todos"] + L.append(f"## To-dos: {td['count']} open ({len(td['pressing'])} pressing, {len(td['long_term'])} long-term)") + L.append(f"**Pressing ({len(td['pressing'])}):**") + for x in td["pressing"][:40]: + L.append(f" - `{x['file']}`:{x['line']} {x['text']}") + L.append(f"**Long-term ({len(td['long_term'])})** — standing checklists, not nagged:") + for x in td["long_term"][:15]: + L.append(f" - `{x['file']}`:{x['line']} {x['text']}") + if len(td["long_term"]) > 15: + L.append(f" … and {len(td['long_term'])-15} more long-term") + L.append("") + + ch = r["category_health"] + if ch["near_id_limit"]: + L.append("## Category health") + for x in ch["near_id_limit"]: + L.append(f" - `{x['category']}` has {x['ids']} IDs (near 10 limit)") + if ch["duplicate_titles"]: + L.append(f" - duplicate ID titles across systems ({len(ch['duplicate_titles'])}): " + + "; ".join(f"{t} {v}" for t, v in list(ch["duplicate_titles"].items())[:10])) + return "\n".join(L) + + +def main(): + r = build_report() + if "--pretty" in sys.argv: + print(pretty(r)) + else: + print(json.dumps(r, indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/vault-topology.py b/vault-topology.py new file mode 100644 index 0000000..68c255e --- /dev/null +++ b/vault-topology.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Generate/refresh the vault topology manifest note (source of truth for both trees).""" +import os, json, sys, datetime +sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) +import vault_lib as vl + +def render(manifest): + ts = datetime.date.today().isoformat() + c = manifest["counts"] + lines = [] + lines.append("---") + lines.append("tags: [meta, topology, jdex]") + lines.append("generated: " + ts) + lines.append("---") + lines.append("# 00.05 Vault topology (machine-readable)") + lines.append("") + lines.append("> [!info] Source of truth") + lines.append("> This note is the **authoritative topology** for the R/J/F/I vault. It is") + lines.append("> regenerated from the live `/vault` structure by `vault-topology.py` and is") + lines.append("> what the `vault` CLI and the nightly Librarian reconcile **both** `/vault`") + lines.append("> (the JDex/map) and `/data` (blob territory) against. Do not hand-edit the") + lines.append("> JSON block below — change the vault via the `vault` CLI and regenerate.") + lines.append("") + lines.append(f"**Live counts:** {c['systems']} systems · {c['areas']} areas · " + f"{c['categories']} categories · {c['ids']} IDs") + lines.append("") + lines.append("## Systems") + for k, v in manifest["systems"].items(): + lines.append(f"- **{k}** — {v}") + lines.append("") + lines.append("## Categories by system") + # group categories under systems + by_sys = {} + for code, name in manifest["categories"].items(): + by_sys.setdefault(code[0], []).append((code, name)) + for s in sorted(by_sys): + lines.append(f"### {s}") + for code, name in sorted(by_sys[s]): + n_ids = sum(1 for idc in manifest["ids"] if idc.startswith(code + ".")) + lines.append(f"- `{code}` {name.split(' ',1)[1] if ' ' in name else name} — {n_ids} IDs") + lines.append("") + lines.append("## Machine-readable topology") + lines.append("") + lines.append(vl.MANIFEST_BEGIN) + lines.append("```json") + lines.append(json.dumps(manifest, indent=2, ensure_ascii=False)) + lines.append("```") + lines.append(vl.MANIFEST_END) + lines.append("") + return "\n".join(lines) + +def main(): + manifest = vl.build_manifest() + content = render(manifest) + dest = os.path.join(vl.VAULT, vl.MANIFEST_REL) + os.makedirs(os.path.dirname(dest), exist_ok=True) + dry = "--execute" not in sys.argv + if dry: + print("DRY-RUN. Would write manifest to:") + print(" " + dest) + print(f" ({manifest['counts']})") + print("--- preview (first 40 lines) ---") + print("\n".join(content.splitlines()[:40])) + else: + with open(dest, "w", encoding="utf-8") as f: + f.write(content) + print("WROTE " + dest) + print(f" {manifest['counts']}") + +if __name__ == "__main__": + main() diff --git a/vault_lib.py b/vault_lib.py new file mode 100644 index 0000000..0c33518 --- /dev/null +++ b/vault_lib.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python3 +""" +vault_lib.py — shared library for Ryan's R/J/F/I Obsidian vault Librarian tooling. + +Single source of truth for: + - tree walking + ID parsing (SYS.AC.ID) + - wikilink extraction / broken-link detection + - the topology manifest (machine-readable JSON block in a 00-area note) + - cross-tree (/vault <-> /data) topology diff + +Used by: + - vault (the CLI: search / rename / create / move IDs across both trees) + - vault-librarian.py (the nightly deterministic report generator) + +Conventions (see ryan-vault-rjfi-migration + obsidian-vault-navigation skills): + VAULT = /vault -> the JDex (markdown map). Obsidian-synced. Authoritative for structure. + DATA = /data -> blob/artifact territory. SMB-served. Mirrors the SYS.AC skeleton, + MINUS `0 System area`, PLUS `_unsorted/` (allowed, no vault counterpart). + Every level is prefixed: `R Ryan / R.1 Life admin / R.11 Identity... / R.11.84 ... / R.11.84 ....md` + Folder-note = a .md whose name == its parent folder name (Folder Notes plugin needs this). +""" +import os +import re +import json + +VAULT = "/vault" +DATA = "/data" + +# Directories never treated as vault structure. +SKIP_DIRS = {".git", ".obsidian"} +# Top-level vault dir that is meta, not a system (unprefixed). +SYSTEM_META = "0 System area" +# /data dirs allowed to exist without a /vault counterpart. +DATA_EXEMPT = {"_unsorted"} + +# --- regexes on folder/leaf basenames --- +SYS_RE = re.compile(r'^([RJFI]) ') # "R Ryan" +AREA_RE = re.compile(r'^([RJFI])\.(\d) ') # "R.1 Life admin" +CAT_RE = re.compile(r'^([RJFI])\.(\d\d) ') # "R.11 Identity & vital records" +ID_RE = re.compile(r'^([RJFI])\.(\d\d)\.(\d\d) ') # "R.11.84 My journal & notes by day" +# extend-notation leaf that never got a real ID, e.g. "25+ Smoker Brisket.md" +EXTEND_RE = re.compile(r'^(\d\d)\+ ') + +WIKILINK_RE = re.compile(r'!?\[\[([^\]]+)\]\]') + + +def parse_id(basename): + """Return (sys, area_digit, cat, id) or None. basename is a folder/leaf name.""" + m = ID_RE.match(basename) + if not m: + return None + return {"sys": m.group(1), "area": m.group(2), "cat": m.group(2) + m.group(3)[:0], "raw": basename} + + +def id_code(basename): + """Return the 'R.11.84' code for a leaf name, or None.""" + m = ID_RE.match(basename) + if not m: + return None + return f"{m.group(1)}.{m.group(2)}.{m.group(3)}" + + +def cat_code(basename): + m = CAT_RE.match(basename) + return f"{m.group(1)}.{m.group(2)}" if m else None + + +def area_code(basename): + m = AREA_RE.match(basename) + return f"{m.group(1)}.{m.group(2)}" if m else None + + +def walk_structure(root, include_meta=False): + """ + Walk a vault-shaped tree. Return dict: + systems: {name} + areas: {code -> relpath} + cats: {code -> relpath} + ids: {code -> relpath} (folder relpaths) + Skips .git/.obsidian. If include_meta False, skips the `0 System area` subtree. + """ + systems, areas, cats, ids = {}, {}, {}, {} + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS] + rel = os.path.relpath(dirpath, root) + top = rel.split(os.sep)[0] + if not include_meta and top == SYSTEM_META: + dirnames[:] = [] + continue + base = os.path.basename(dirpath) + if ID_RE.match(base): + ids[id_code(base)] = rel + elif CAT_RE.match(base): + cats[cat_code(base)] = rel + elif AREA_RE.match(base): + areas[area_code(base)] = rel + elif SYS_RE.match(base) and os.sep not in rel: + systems[base[0]] = rel + return {"systems": systems, "areas": areas, "cats": cats, "ids": ids} + + +ARCHIVE_SUBPATH = os.path.join(SYSTEM_META, "09 Archive of pre-RJFI tree") + + +def is_archived(relpath): + """True if a vault-relative path is inside the frozen pre-RJFI archive.""" + return relpath.startswith(ARCHIVE_SUBPATH) + + +def all_notes(root, include_meta=True, include_archive=True): + """Yield absolute paths of every .md note (skips .git/.obsidian). + include_archive=False skips the frozen `09 Archive of pre-RJFI tree` subtree.""" + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS] + rel = os.path.relpath(dirpath, root) + if not include_archive and is_archived(rel): + dirnames[:] = [] + continue + for f in filenames: + if f.endswith(".md"): + yield os.path.join(dirpath, f) + + +def is_folder_note(path): + """A note whose filename stem == its parent directory name.""" + stem = os.path.splitext(os.path.basename(path))[0] + parent = os.path.basename(os.path.dirname(path)) + return stem == parent + + +def build_title_index(root): + """ + Map every resolvable wikilink target -> abs path. + Obsidian resolves by basename (title) AND by full relative path. + """ + idx = {} + for p in all_notes(root): + stem = os.path.splitext(os.path.basename(p))[0] + rel = os.path.relpath(p, root) + relnoext = os.path.splitext(rel)[0] + idx.setdefault(stem, p) + idx[relnoext] = p # full-path form always wins if explicit + return idx + + +def extract_links(text): + """Return list of (raw_target, cleaned_target). Strips #heading and |alias.""" + out = [] + for m in WIKILINK_RE.finditer(text): + raw = m.group(1) + cleaned = raw.split("|")[0].split("#")[0].strip() + out.append((raw, cleaned)) + return out + + +# link targets we deliberately skip (blobs, code, doc examples) +_BLOB_EXT = re.compile(r'\.(jpg|jpeg|png|gif|webp|pdf|svg|opus|mp3|mp4|mov|3gp|zip|xlsx?|docx?|pptx?|csv|heic)$', re.I) +_NOISE = ("this.file", "{{", "...", '"') + + +def link_is_checkable(target): + if not target: + return False + if _BLOB_EXT.search(target): + return False + if any(n in target for n in _NOISE): + return False + return True + + +def find_broken_links(root, include_archive=False): + """ + Return list of dicts: {file, target, guess} + guess = an existing note title that likely IS the intended target, else None. + Guess strategies (in order): case-insensitive title match; then match on the + NN.NN ID-number tail across systems (catches migration carry-forwards like + [[11.30 Mental health]] -> R.12.30, and stale un-prefixed self-links [[13.05 ..]] -> F.13.05). + Skips the frozen 09-archive subtree by default. + """ + idx = build_title_index(root) + low = {} + for k in idx: + low.setdefault(k.lower(), k) + # index existing notes by their NN.NN id-number tail -> [stems] + idnum_re = re.compile(r'(\d\d\.\d\d)') + by_idnum = {} + for k in idx: + m = re.match(r'^[RJFI]?\.?(\d\d\.\d\d)', k) + if m: + by_idnum.setdefault(m.group(1), []).append(k) + broken = [] + for p in all_notes(root, include_archive=include_archive): + try: + text = open(p, encoding="utf-8").read() + except Exception: + continue + for raw, cleaned in extract_links(text): + if not link_is_checkable(cleaned): + continue + if cleaned in idx: + continue + base = cleaned.split("/")[-1] + if base in idx: + continue + guess = low.get(cleaned.lower()) or low.get(base.lower()) + if not guess: + m = re.match(r'^[RJFI]?\.?(\d\d\.\d\d)', cleaned) + if m and m.group(1) in by_idnum: + cands = by_idnum[m.group(1)] + if len(cands) == 1: + guess = cands[0] + broken.append({ + "file": os.path.relpath(p, root), + "target": cleaned, + "guess": guess, + }) + return broken + + +def topology_diff(vault_root=VAULT, data_root=DATA): + """ + Compare the SYS.AC.ID skeleton of /vault (authoritative) vs /data. + /data skips `0 System area` and is allowed `_unsorted`. + Returns dict of missing/orphan at each level. + """ + v = walk_structure(vault_root, include_meta=False) + d = walk_structure(data_root, include_meta=False) + out = {} + for level in ("systems", "areas", "cats", "ids"): + vk = set(v[level].keys()) + dk = set(d[level].keys()) + out[level] = { + "missing_in_data": sorted(vk - dk), # vault has it, /data lacks a home + "orphan_in_data": sorted(dk - vk), # /data has it, no vault note + } + return out + + +# ---- topology manifest (the 00-area source-of-truth note) ---- +MANIFEST_REL = "0 System area/00 System category/00.05 Vault topology (machine-readable)/00.05 Vault topology (machine-readable).md" +MANIFEST_BEGIN = "<!-- VAULT-TOPOLOGY-JSON:BEGIN -->" +MANIFEST_END = "<!-- VAULT-TOPOLOGY-JSON:END -->" + + +def build_manifest(vault_root=VAULT): + """Build the topology manifest dict from the live vault (authoritative).""" + s = walk_structure(vault_root, include_meta=True) + return { + "systems": {k: os.path.basename(v) for k, v in sorted(s["systems"].items())}, + "areas": {k: os.path.basename(v) for k, v in sorted(s["areas"].items())}, + "categories": {k: os.path.basename(v) for k, v in sorted(s["cats"].items())}, + "ids": {k: os.path.basename(v) for k, v in sorted(s["ids"].items())}, + "counts": { + "systems": len(s["systems"]), + "areas": len(s["areas"]), + "categories": len(s["cats"]), + "ids": len(s["ids"]), + }, + } + + +def read_manifest(vault_root=VAULT): + """Parse the JSON block out of the manifest note. Return dict or None.""" + p = os.path.join(vault_root, MANIFEST_REL) + if not os.path.exists(p): + return None + text = open(p, encoding="utf-8").read() + i = text.find(MANIFEST_BEGIN) + j = text.find(MANIFEST_END) + if i < 0 or j < 0: + return None + block = text[i + len(MANIFEST_BEGIN):j] + block = block.strip().strip("`").strip() + if block.startswith("json"): + block = block[4:].strip() + try: + return json.loads(block) + except Exception: + return None + + +if __name__ == "__main__": + import sys + s = walk_structure(VAULT, include_meta=True) + print(json.dumps({k: len(v) for k, v in s.items()}, indent=2)) diff --git a/vault_ops.py b/vault_ops.py new file mode 100644 index 0000000..287fb4a --- /dev/null +++ b/vault_ops.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +""" +vault_ops.py — write operations for the `vault` CLI. +Clobber-safe by construction: refuse if destination exists, refuse if source missing. +Every op is dry-run unless execute=True. Operates on BOTH /vault and /data + manifest. +""" +import os, re, sys, subprocess +_SELF_DIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, _SELF_DIR) +import vault_lib as vl + +WIKILINK_RE = re.compile(r'(!?\[\[)([^\]#|]+)((?:#[^\]|]+)?(?:\|[^\]]+)?)(\]\])') + + +def _die(m): + print("ERROR: " + m, file=sys.stderr); sys.exit(1) + + +def _sys_root(letter): + s = vl.walk_structure(vl.VAULT, include_meta=True)["systems"] + if letter not in s: + _die(f"unknown system '{letter}'") + return os.path.join(vl.VAULT, s[letter]) + + +def _find_id_folder(root, code): + for dp, dns, fns in os.walk(root): + dns[:] = [d for d in dns if d not in vl.SKIP_DIRS] + if vl.id_code(os.path.basename(dp)) == code: + return dp + return None + + +def _find_cat_folder(root, catcode): + for dp, dns, fns in os.walk(root): + dns[:] = [d for d in dns if d not in vl.SKIP_DIRS] + if vl.cat_code(os.path.basename(dp)) == catcode: + return dp + return None + + +def _rewrite_links(old_stem, new_stem, execute): + """Rewrite [[old_stem]] -> [[new_stem]] vault-wide. Returns count.""" + n = 0 + for p in vl.all_notes(vl.VAULT): + try: + txt = open(p, encoding="utf-8").read() + except Exception: + continue + def repl(m): + nonlocal n + if m.group(2).strip() == old_stem: + n += 1 + return f"{m.group(1)}{new_stem}{m.group(3)}{m.group(4)}" + return m.group(0) + new = WIKILINK_RE.sub(repl, txt) + if new != txt and execute: + open(p, "w", encoding="utf-8").write(new) + return n + + +def _git_commit(msg, execute): + if not execute: + return + subprocess.run(["git", "-C", vl.VAULT, "add", "-A"], check=False) + subprocess.run(["git", "-C", vl.VAULT, "commit", "-m", msg], check=False) + + +def op_create(argv, execute): + if len(argv) < 2: + _die('usage: vault create <ID> "<title>"') + code, title = argv[0], argv[1] + if not vl.ID_RE.match(code + " x"): + _die(f"'{code}' is not a valid SYS.AC.ID code") + root = _sys_root(code[0]) + catcode = code[:code.rindex(".")] + catfolder = _find_cat_folder(root, catcode) + if not catfolder: + _die(f"category {catcode} does not exist — create it first") + if _find_id_folder(root, code): + _die(f"CLOBBER: ID {code} already exists") + leaf = f"{code} {title}" + vpath = os.path.join(catfolder, leaf) + vnote = os.path.join(vpath, leaf + ".md") + drel = os.path.relpath(vpath, vl.VAULT) + dpath = os.path.join(vl.DATA, drel) + print(f"CREATE /vault: {os.path.relpath(vpath, vl.VAULT)}") + print(f" folder-note: {leaf}.md") + print(f" /data: {drel}") + if os.path.exists(dpath): + _die(f"CLOBBER: /data path already exists: {drel}") + if execute: + os.makedirs(vpath) + with open(vnote, "w", encoding="utf-8") as f: + f.write(f"# {leaf}\n") + os.makedirs(dpath) + _git_commit(f"vault create {code} {title}", execute) + print("EXECUTED.") + else: + print("DRY-RUN — nothing changed. Add --execute.") + + +def op_move(argv, execute): + if len(argv) < 2: + _die("usage: vault move <oldID> <newID>") + old, new = argv[0], argv[1] + root = _sys_root(old[0]) + oldf = _find_id_folder(root, old) + if not oldf: + _die(f"source ID {old} not found") + newroot = _sys_root(new[0]) + if _find_id_folder(newroot, new): + _die(f"CLOBBER: destination ID {new} already exists") + newcat = new[:new.rindex(".")] + newcatf = _find_cat_folder(newroot, newcat) + if not newcatf: + _die(f"destination category {newcat} does not exist") + old_base = os.path.basename(oldf) + title = old_base.split(" ", 1)[1] if " " in old_base else "" + new_base = f"{new} {title}".strip() + new_folder = os.path.join(newcatf, new_base) + old_stem, new_stem = old_base, new_base + old_drel = os.path.relpath(oldf, vl.VAULT) + new_drel = os.path.relpath(new_folder, vl.VAULT) + print(f"MOVE {old_base} -> {new_base}") + print(f" /vault: {old_drel} -> {new_drel}") + print(f" /data: {old_drel} -> {new_drel}") + print(f" backlink rewrite: [[{old_stem}]] -> [[{new_stem}]]") + n = _rewrite_links(old_stem, new_stem, execute) + print(f" ({n} backlink(s))") + if execute: + # rename folder-note first, then folder + oldnote = os.path.join(oldf, old_base + ".md") + if os.path.exists(oldnote): + os.rename(oldnote, os.path.join(oldf, new_base + ".md")) + os.rename(oldf, new_folder) + olddata = os.path.join(vl.DATA, old_drel) + if os.path.isdir(olddata): + os.makedirs(os.path.dirname(os.path.join(vl.DATA, new_drel)), exist_ok=True) + os.rename(olddata, os.path.join(vl.DATA, new_drel)) + _regen_manifest(execute) + _git_commit(f"vault move {old} -> {new}", execute) + print("EXECUTED.") + else: + print("DRY-RUN — nothing changed. Add --execute.") + + +def op_rename(argv, execute): + if len(argv) < 2: + _die('usage: vault rename <ID> "<new title>"') + code, title = argv[0], argv[1] + root = _sys_root(code[0]) + f = _find_id_folder(root, code) + if not f: + _die(f"ID {code} not found") + old_base = os.path.basename(f) + new_base = f"{code} {title}" + if old_base == new_base: + _die("new title identical to current") + parent = os.path.dirname(f) + if os.path.exists(os.path.join(parent, new_base)): + _die(f"CLOBBER: {new_base} already exists") + print(f"RENAME {old_base} -> {new_base}") + n = _rewrite_links(old_base, new_base, execute) + print(f" ({n} backlink(s))") + if execute: + oldnote = os.path.join(f, old_base + ".md") + if os.path.exists(oldnote): + os.rename(oldnote, os.path.join(f, new_base + ".md")) + newf = os.path.join(parent, new_base) + os.rename(f, newf) + drel_old = os.path.relpath(f, vl.VAULT) + drel_new = os.path.relpath(newf, vl.VAULT) + od = os.path.join(vl.DATA, drel_old) + if os.path.isdir(od): + os.rename(od, os.path.join(vl.DATA, drel_new)) + _regen_manifest(execute) + _git_commit(f"vault rename {code}", execute) + print("EXECUTED.") + else: + print("DRY-RUN — nothing changed. Add --execute.") + + +def op_sync_data(argv, execute): + """Create any /data skeleton dir that a /vault ID/cat/area lacks. Never deletes.""" + v = vl.walk_structure(vl.VAULT, include_meta=False) + made = [] + for level in ("areas", "cats", "ids"): + for code, rel in sorted(v[level].items()): + dpath = os.path.join(vl.DATA, rel) + if not os.path.isdir(dpath): + made.append(rel) + if execute: + os.makedirs(dpath, exist_ok=True) + print(f"sync-data: {len(made)} /data dir(s) {'created' if execute else 'to create'}") + for m in made[:40]: + print(" + " + m) + if len(made) > 40: + print(f" ... and {len(made)-40} more") + if execute: + print("EXECUTED.") + else: + print("DRY-RUN — nothing changed. Add --execute.") + + +def _regen_manifest(execute): + if execute: + subprocess.run([sys.executable, os.path.join(_SELF_DIR, "vault-topology.py"), "--execute"], check=False) + + +def op_manifest(argv, execute): + subprocess.run([sys.executable, os.path.join(_SELF_DIR, "vault-topology.py")] + + (["--execute"] if execute else []), check=False) + + +def run(cmd, argv, execute): + { + "create": op_create, + "move": op_move, + "rename": op_rename, + "sync-data": op_sync_data, + "manifest": op_manifest, + }[cmd](argv, execute)