Compare commits
18
Commits
4de2d92bbc
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a5814a37af | ||
|
|
1938b889e2 | ||
|
|
cf6196d525 | ||
|
|
77e4271660 | ||
|
|
8ffba8f7ee | ||
|
|
49c46042b5 | ||
|
|
90fbc240af | ||
|
|
d89aeb7d26 | ||
|
|
9f3a99a42e | ||
|
|
8e8ad466a8 | ||
|
|
57776d46fa | ||
|
|
069552d1b4 | ||
|
|
abf9569a9a | ||
|
|
a347c55479 | ||
|
|
87bb628ad3 | ||
|
|
d81d156b79 | ||
|
|
b665413630 | ||
|
|
e535ee3608 |
@@ -0,0 +1,2 @@
|
||||
__pycache__/
|
||||
*.pyc
|
||||
Binary file not shown.
@@ -0,0 +1,33 @@
|
||||
{
|
||||
description = "vault CLI + Librarian tooling for Ryan's R/J/F/I Obsidian vault";
|
||||
|
||||
inputs = {
|
||||
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
|
||||
};
|
||||
|
||||
outputs = { self, nixpkgs }:
|
||||
let
|
||||
systems = [ "x86_64-linux" "aarch64-linux" ];
|
||||
forAllSystems = nixpkgs.lib.genAttrs systems;
|
||||
in
|
||||
{
|
||||
packages = forAllSystems (system:
|
||||
let
|
||||
pkgs = nixpkgs.legacyPackages.${system};
|
||||
vault-cli = pkgs.callPackage ./package.nix { };
|
||||
in
|
||||
{
|
||||
default = vault-cli;
|
||||
vault-cli = vault-cli;
|
||||
});
|
||||
|
||||
# Convenience module for consumers (e.g. nixos-config) that just want
|
||||
# `vault` on $PATH for hosts playing a vault role. Mirrors the shape
|
||||
# nixos-config's modules/hosts/core/vault.nix expects: import this and
|
||||
# add `environment.systemPackages = [ pkgs.vault-cli ]` (or reference
|
||||
# `self.packages.${system}.default` directly via a flake input).
|
||||
nixosModules.default = { pkgs, ... }: {
|
||||
environment.systemPackages = [ self.packages.${pkgs.system}.default ];
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Add a short-name alias (ID prefix stripped) to every ID-prefixed note's
|
||||
frontmatter, so Obsidian's alias-autocomplete can produce clean [[ID|Name]]
|
||||
links without the user typing the ID. Idempotent: skips notes that already
|
||||
have that alias or non-ID-prefixed notes.
|
||||
"""
|
||||
import os, re, sys
|
||||
|
||||
VAULT = "/vault"
|
||||
# Matches "F.50.12 Jen", "R.31.10 Hindi", "F.22.11 Places We've Slept" etc.
|
||||
ID_TITLE_RE = re.compile(r'^([RJFI](?:\.\d+){1,3})\s+(.+)$')
|
||||
|
||||
FRONTMATTER_RE = re.compile(r'^---\n(.*?)\n---\n', re.DOTALL)
|
||||
|
||||
def split_frontmatter(text):
|
||||
m = FRONTMATTER_RE.match(text)
|
||||
if m:
|
||||
return m.group(1), text[m.end():]
|
||||
return None, text
|
||||
|
||||
def add_alias(fm_text, alias):
|
||||
lines = fm_text.split("\n") if fm_text else []
|
||||
# find aliases: line (list style) or aliases: [a, b]
|
||||
alias_line_idx = None
|
||||
for i, l in enumerate(lines):
|
||||
if re.match(r'^aliases\s*:', l):
|
||||
alias_line_idx = i
|
||||
break
|
||||
if alias_line_idx is None:
|
||||
lines.append(f"aliases:\n - {alias}")
|
||||
return "\n".join(lines)
|
||||
line = lines[alias_line_idx]
|
||||
# inline list form: aliases: [a, b]
|
||||
m = re.match(r'^aliases\s*:\s*\[(.*)\]\s*$', line)
|
||||
if m:
|
||||
existing = [x.strip().strip('"\'') for x in m.group(1).split(",") if x.strip()]
|
||||
if alias in existing:
|
||||
return None # already present
|
||||
existing.append(alias)
|
||||
lines[alias_line_idx] = "aliases: [" + ", ".join(existing) + "]"
|
||||
return "\n".join(lines)
|
||||
# block list form: aliases:\n - a\n - b
|
||||
if line.strip() == "aliases:" or re.match(r'^aliases\s*:\s*$', line):
|
||||
j = alias_line_idx + 1
|
||||
existing = []
|
||||
while j < len(lines) and re.match(r'^\s*-\s*', lines[j]):
|
||||
existing.append(lines[j].strip()[2:].strip().strip('"\''))
|
||||
j += 1
|
||||
if alias in existing:
|
||||
return None
|
||||
lines.insert(j, f" - {alias}")
|
||||
return "\n".join(lines)
|
||||
# single scalar alias: aliases: Foo
|
||||
scalar = line.split(":", 1)[1].strip().strip('"\'')
|
||||
if scalar == alias:
|
||||
return None
|
||||
lines[alias_line_idx] = "aliases:\n - " + scalar + f"\n - {alias}"
|
||||
return "\n".join(lines)
|
||||
|
||||
def main():
|
||||
execute = "--execute" in sys.argv
|
||||
changed = 0
|
||||
skipped_no_id = 0
|
||||
skipped_has_alias = 0
|
||||
errors = []
|
||||
for dp, _, fns in os.walk(VAULT):
|
||||
if ".git" in dp.split(os.sep):
|
||||
continue
|
||||
for fn in fns:
|
||||
if not fn.lower().endswith(".md"):
|
||||
continue
|
||||
base = fn[:-3]
|
||||
m = ID_TITLE_RE.match(base)
|
||||
if not m:
|
||||
skipped_no_id += 1
|
||||
continue
|
||||
alias = m.group(2)
|
||||
p = os.path.join(dp, fn)
|
||||
try:
|
||||
with open(p, "r", encoding="utf-8", errors="ignore") as f:
|
||||
text = f.read()
|
||||
except Exception as e:
|
||||
errors.append((p, str(e)))
|
||||
continue
|
||||
fm, rest = split_frontmatter(text)
|
||||
new_fm = add_alias(fm or "", alias)
|
||||
if new_fm is None:
|
||||
skipped_has_alias += 1
|
||||
continue
|
||||
new_text = f"---\n{new_fm}\n---\n\n{rest.lstrip(chr(10))}" if fm is None else f"---\n{new_fm}\n---\n{rest}"
|
||||
changed += 1
|
||||
if execute:
|
||||
with open(p, "w", encoding="utf-8") as f:
|
||||
f.write(new_text)
|
||||
|
||||
print(f"Notes with ID-prefixed title: {changed + skipped_has_alias}")
|
||||
print(f" Would add alias: {changed}")
|
||||
print(f" Already had alias: {skipped_has_alias}")
|
||||
print(f"Notes without ID prefix (skipped, not applicable): {skipped_no_id}")
|
||||
if errors:
|
||||
print(f"Errors: {len(errors)}")
|
||||
for p, e in errors[:10]:
|
||||
print(f" {p}: {e}")
|
||||
if not execute:
|
||||
print("\n(dry run -- pass --execute to write changes)")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Scan /vault for [[wikilinks]] and report which targets don't resolve
|
||||
to an actual note. Obsidian resolves links by basename match anywhere
|
||||
in the vault (unless the link includes a path), so we build an index
|
||||
of all note basenames (and their full relative paths) first.
|
||||
"""
|
||||
import os, re, json
|
||||
from collections import defaultdict
|
||||
|
||||
VAULT = "/vault"
|
||||
LINK_RE = re.compile(r'\[\[([^\]|#]+)(?:#[^\]|]*)?(?:\|[^\]]*)?\]\]')
|
||||
|
||||
def all_notes():
|
||||
idx = defaultdict(list) # basename (no ext, lowercase) -> [full paths]
|
||||
all_paths = []
|
||||
for dp, dirnames, fns in os.walk(VAULT):
|
||||
if ".git" in dp.split(os.sep):
|
||||
continue
|
||||
for fn in fns:
|
||||
if fn.lower().endswith(".md"):
|
||||
p = os.path.join(dp, fn)
|
||||
base = fn[:-3]
|
||||
idx[base.lower()].append(p)
|
||||
all_paths.append(p)
|
||||
return idx, all_paths
|
||||
|
||||
def resolve(link_target, index, current_dir):
|
||||
# Obsidian: link may be a bare name, or a relative/vault path (with /)
|
||||
target = link_target.strip()
|
||||
if "/" in target:
|
||||
# path-style link; try resolving relative to vault root or current dir
|
||||
candidates = [
|
||||
os.path.join(VAULT, target + ".md"),
|
||||
os.path.join(VAULT, target),
|
||||
os.path.join(current_dir, target + ".md"),
|
||||
os.path.join(current_dir, target),
|
||||
]
|
||||
for c in candidates:
|
||||
if os.path.isfile(c):
|
||||
return True
|
||||
# also try basename match as fallback
|
||||
base = os.path.basename(target)
|
||||
return base.lower() in index
|
||||
else:
|
||||
return target.lower() in index
|
||||
|
||||
def main():
|
||||
index, all_paths = all_notes()
|
||||
unresolved = defaultdict(list) # target -> [source files]
|
||||
total_links = 0
|
||||
for p in all_paths:
|
||||
try:
|
||||
with open(p, "r", encoding="utf-8", errors="ignore") as f:
|
||||
text = f.read()
|
||||
except Exception:
|
||||
continue
|
||||
for m in LINK_RE.finditer(text):
|
||||
target = m.group(1)
|
||||
total_links += 1
|
||||
if not resolve(target, index, os.path.dirname(p)):
|
||||
unresolved[target].append(os.path.relpath(p, VAULT))
|
||||
|
||||
print(f"Total wikilinks scanned: {total_links}")
|
||||
print(f"Unique unresolved targets: {len(unresolved)}")
|
||||
print(f"Total unresolved link occurrences: {sum(len(v) for v in unresolved.values())}")
|
||||
|
||||
# dump full detail to json for further processing
|
||||
out = {t: srcs for t, srcs in unresolved.items()}
|
||||
with open("/tmp/unresolved_links.json", "w") as f:
|
||||
json.dump(out, f, indent=2)
|
||||
print("Full detail written to /tmp/unresolved_links.json")
|
||||
|
||||
# sorted by occurrence count desc
|
||||
by_count = sorted(unresolved.items(), key=lambda kv: -len(kv[1]))
|
||||
print("\n=== TOP 40 unresolved targets by occurrence count ===")
|
||||
for target, srcs in by_count[:40]:
|
||||
print(f" {len(srcs):4d}x [[{target}]]")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Apply the two link-fix passes across /vault:
|
||||
1. fix_map: [[OldTarget]] -> [[NewNote|OldTarget]] (preserves display text)
|
||||
Also handles piped/heading variants: [[OldTarget|Display]] -> [[NewNote|Display]]
|
||||
[[OldTarget#Heading]] -> [[NewNote#Heading]]
|
||||
2. unlink_targets: [[Target]] -> Target (plain text, drop brackets)
|
||||
Also handles piped: [[Target|Display]] -> Display
|
||||
"""
|
||||
import os, re, json, sys
|
||||
|
||||
VAULT = "/vault"
|
||||
|
||||
with open("/tmp/link_fix_map.json") as f:
|
||||
FIX_MAP = json.load(f)
|
||||
with open("/tmp/unlink_targets.json") as f:
|
||||
UNLINK = set(json.load(f))
|
||||
|
||||
# Full link regex capturing target, optional #heading, optional |display
|
||||
LINK_RE = re.compile(r'\[\[([^\]|#]+)(#[^\]|]*)?(\|[^\]]*)?\]\]')
|
||||
|
||||
def process_text(text):
|
||||
changed = False
|
||||
def repl(m):
|
||||
nonlocal changed
|
||||
target = m.group(1)
|
||||
heading = m.group(2) or ""
|
||||
pipe_display = m.group(3) # includes leading |
|
||||
target_stripped = target.strip()
|
||||
|
||||
if target_stripped in FIX_MAP:
|
||||
new_target = FIX_MAP[target_stripped]
|
||||
changed = True
|
||||
if pipe_display:
|
||||
# keep existing custom display text
|
||||
return f"[[{new_target}{heading}{pipe_display}]]"
|
||||
else:
|
||||
# preserve original text as display so prose doesn't change visually
|
||||
return f"[[{new_target}{heading}|{target_stripped}]]"
|
||||
|
||||
if target_stripped in UNLINK:
|
||||
changed = True
|
||||
if pipe_display:
|
||||
display = pipe_display[1:] # strip leading |
|
||||
return display
|
||||
else:
|
||||
return target_stripped
|
||||
|
||||
return m.group(0) # unchanged
|
||||
|
||||
new_text = LINK_RE.sub(repl, text)
|
||||
return new_text, changed
|
||||
|
||||
def main():
|
||||
execute = "--execute" in sys.argv
|
||||
files_changed = 0
|
||||
total_fix_applied = 0
|
||||
total_unlink_applied = 0
|
||||
|
||||
for dp, _, fns in os.walk(VAULT):
|
||||
if ".git" in dp.split(os.sep):
|
||||
continue
|
||||
for fn in fns:
|
||||
if not fn.lower().endswith(".md"):
|
||||
continue
|
||||
p = os.path.join(dp, fn)
|
||||
try:
|
||||
with open(p, "r", encoding="utf-8", errors="ignore") as f:
|
||||
text = f.read()
|
||||
except Exception:
|
||||
continue
|
||||
new_text, changed = process_text(text)
|
||||
if changed:
|
||||
files_changed += 1
|
||||
if execute:
|
||||
with open(p, "w", encoding="utf-8") as f:
|
||||
f.write(new_text)
|
||||
|
||||
print(f"Files changed: {files_changed}")
|
||||
if not execute:
|
||||
print("(dry run -- pass --execute to write)")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sort /data/_unsorted (old pre-RJFI JD tree) into /data/R,J,F,I storage tree.
|
||||
|
||||
Dry-run by default; pass --execute to actually move files.
|
||||
"""
|
||||
import os, shutil, sys, csv
|
||||
|
||||
UNSORTED = "/data/_unsorted"
|
||||
DATA = "/data"
|
||||
|
||||
# old_prefix -> new_prefix (folder-level rename/move rules)
|
||||
# Order matters: longer/more specific prefixes first.
|
||||
MAPPING = [
|
||||
# --- WhatsApp exports: wholesale move, old->new folder rename ---
|
||||
("1 Life admin/11 Me & other living things/11.54 Whatsapp exports",
|
||||
"R Ryan/R.1 Life admin/R.11 Identity & vital records/R.11.54 Whatsapp exports"),
|
||||
|
||||
# --- 12 Home, house, and transport -> split F.11 (home) / F.12 (vehicles) ---
|
||||
("1 Life admin/12 Home, house, and transport/12.12 Home insurance & claims",
|
||||
"F Family/F.1 Home, household & finances/F.11 Home & property/F.11.12 Home insurance & claims"),
|
||||
("1 Life admin/12 Home, house, and transport/12.21 Electricity, gas, & water",
|
||||
"F Family/F.1 Home, household & finances/F.11 Home & property/F.11.21 Electricity, gas, & water"),
|
||||
("1 Life admin/12 Home, house, and transport/12.23 All other utilities & services",
|
||||
"F Family/F.1 Home, household & finances/F.11 Home & property/F.11.23 All other utilities & services"),
|
||||
("1 Life admin/12 Home, house, and transport/12.24 Renovations & improvements",
|
||||
"F Family/F.1 Home, household & finances/F.11 Home & property/F.11.24 Renovations & improvements"),
|
||||
("1 Life admin/12 Home, house, and transport/12.25 House repairs, maintenance, & upkeep",
|
||||
"F Family/F.1 Home, household & finances/F.11 Home & property/F.11.24 Renovations & improvements"),
|
||||
("1 Life admin/12 Home, house, and transport/12.61 Move out of 6 Dorothy",
|
||||
"F Family/F.1 Home, household & finances/F.11 Home & property/F.11.61 Move out of 6 Dorothy"),
|
||||
("1 Life admin/12 Home, house, and transport/12.30 Vehicles",
|
||||
"F Family/F.1 Home, household & finances/F.12 Vehicles/F.12.30 Vehicles"),
|
||||
("1 Life admin/12 Home, house, and transport/12.32 Motor vehicle auto insurance & claims",
|
||||
"F Family/F.1 Home, household & finances/F.12 Vehicles/F.12.32 Motor vehicle auto insurance & claims"),
|
||||
("1 Life admin/12 Home, house, and transport/12.33 Car repairs, maintenance, & upkeep",
|
||||
"F Family/F.1 Home, household & finances/F.12 Vehicles/F.12.33 Car repairs, maintenance, & upkeep"),
|
||||
|
||||
# --- 13 Money -> split R.13 (solo) / F.13 (joint); tax + financial insurance are joint per design doc ---
|
||||
("1 Life admin/13 Money earned, spent, saved, & owed/13.54 Tax returns & accounting",
|
||||
"F Family/F.1 Home, household & finances/F.13 Household finances/F.13.05 Tax returns & accounting"),
|
||||
("1 Life admin/13 Money earned, spent, saved, & owed/13.12 Financial insurance & claims",
|
||||
"F Family/F.1 Home, household & finances/F.13 Household finances/F.13.12 Financial insurance & claims"),
|
||||
("1 Life admin/13 Money earned, spent, saved, & owed/13.44 Cryptocurrency",
|
||||
"R Ryan/R.1 Life admin/R.13 Personal finance/R.13.44 Cryptocurrency"),
|
||||
|
||||
# --- 11 Me & other living things -> mostly R (Ryan personal) ---
|
||||
("1 Life admin/11 Me & other living things/11.11 Birth certificate & proof of name",
|
||||
"R Ryan/R.1 Life admin/R.11 Identity & vital records/R.11.11 Birth certificate & proof of name"),
|
||||
("1 Life admin/11 Me & other living things/11.14 Licenses",
|
||||
"R Ryan/R.1 Life admin/R.11 Identity & vital records/R.11.14 Licenses"),
|
||||
("1 Life admin/11 Me & other living things/11.16 Legal documents & certificates",
|
||||
"R Ryan/R.1 Life admin/R.11 Identity & vital records/R.11.16 Legal documents & certificates"),
|
||||
("1 Life admin/11 Me & other living things/11.28 Reproductive health",
|
||||
"R Ryan/R.1 Life admin/R.12 Health & body/R.12.28 Reproductive health"),
|
||||
("1 Life admin/11 Me & other living things/11.83 My library",
|
||||
"R Ryan/R.1 Life admin/R.11 Identity & vital records/R.11.83 Ryan's library"),
|
||||
|
||||
# --- 15 Travel -> R.15 Travel (solo trips; matches exactly) ---
|
||||
("1 Life admin/15 Travel, events, & entertainment/15.41 All short trips",
|
||||
"R Ryan/R.1 Life admin/R.15 Travel/R.15.41 All short trips"),
|
||||
|
||||
# --- 2 Career -> R.2 Career, ~1:1 ---
|
||||
("2 Career/21 Perfect Day", "R Ryan/R.2 Career/R.21 Perfect Day"),
|
||||
("2 Career/22 Sejour", "R Ryan/R.2 Career/R.22 Sejour"),
|
||||
("2 Career/23 Consulting", "R Ryan/R.2 Career/R.23 Consulting"),
|
||||
("2 Career/24 Stanford", "R Ryan/R.2 Career/R.24 Stanford"),
|
||||
("2 Career/29 Old applications & archives", "R Ryan/R.2 Career/R.29 Old applications & archives"),
|
||||
|
||||
# --- 3 Hobbies -> R.3 Hobbies, ~1:1 ---
|
||||
("3 Hobbies & interests/31 Language learning", "R Ryan/R.3 Hobbies & interests/R.31 Language learning"),
|
||||
("3 Hobbies & interests/36 Projects", "R Ryan/R.3 Hobbies & interests/R.36 Projects"),
|
||||
|
||||
# --- 4 Flying -> R.4 Flying, ~1:1 ---
|
||||
("4 Flying/41 Certification", "R Ryan/R.4 Flying/R.41 Certification, legal, & insurance"),
|
||||
("4 Flying/43 Resources", "R Ryan/R.4 Flying/R.43 Resources"),
|
||||
]
|
||||
|
||||
# Items needing human judgment - do NOT auto-move, just report.
|
||||
FLAG_FOR_REVIEW = [
|
||||
"1 Life admin/10 Management of area 10-19", # SMS backups - unclear whose/where
|
||||
"1 Life admin/11 Me & other living things/11.43 My family", # Mom's book proposal
|
||||
"1 Life admin/11 Me & other living things/11.09 Archive for category 11", # AI deck for Jen
|
||||
"1 Life admin/11 Me & other living things/11.71 My career sales pitch",
|
||||
"1 Life admin/13 Money earned, spent, saved, & owed/13.41 Bank accounts", # solo or joint?
|
||||
"1 Life admin/13 Money earned, spent, saved, & owed/13.01 Inbox for category 13",
|
||||
"1 Life admin/12 Home, house, and transport/12.01 Inbox for category 12",
|
||||
"1 Life admin/14 My digital life/14.14 My data storage & backups",
|
||||
"3 Hobbies & interests/30 Management for Hobbies",
|
||||
]
|
||||
|
||||
|
||||
def find_files(root):
|
||||
for dirpath, _, filenames in os.walk(root):
|
||||
for fn in filenames:
|
||||
yield os.path.join(dirpath, fn)
|
||||
|
||||
|
||||
def plan_moves():
|
||||
moves = [] # (src, dst)
|
||||
unmatched = []
|
||||
for src in find_files(UNSORTED):
|
||||
rel = os.path.relpath(src, UNSORTED)
|
||||
matched = False
|
||||
for old_prefix, new_prefix in MAPPING:
|
||||
if rel == old_prefix or rel.startswith(old_prefix + os.sep):
|
||||
suffix = rel[len(old_prefix):]
|
||||
dst = os.path.join(DATA, new_prefix.replace("/", os.sep) + suffix)
|
||||
moves.append((src, dst))
|
||||
matched = True
|
||||
break
|
||||
if not matched:
|
||||
is_flagged = any(rel.startswith(f) for f in FLAG_FOR_REVIEW)
|
||||
unmatched.append((rel, is_flagged))
|
||||
return moves, unmatched
|
||||
|
||||
|
||||
def main():
|
||||
execute = "--execute" in sys.argv
|
||||
moves, unmatched = plan_moves()
|
||||
|
||||
print(f"=== PLANNED MOVES: {len(moves)} files ===")
|
||||
by_target = {}
|
||||
for src, dst in moves:
|
||||
key = os.path.dirname(dst)
|
||||
by_target[key] = by_target.get(key, 0) + 1
|
||||
for k, v in sorted(by_target.items()):
|
||||
print(f" {v:5d} -> {k}")
|
||||
|
||||
print(f"\n=== FLAGGED FOR REVIEW (not moved): {sum(1 for _, f in unmatched if f)} files ===")
|
||||
flagged_dirs = {}
|
||||
for rel, is_flagged in unmatched:
|
||||
if is_flagged:
|
||||
d = os.path.dirname(rel)
|
||||
flagged_dirs[d] = flagged_dirs.get(d, 0) + 1
|
||||
for k, v in sorted(flagged_dirs.items()):
|
||||
print(f" {v:5d} :: {k}")
|
||||
|
||||
print(f"\n=== TRULY UNMATCHED (no mapping rule, not flagged): {sum(1 for _, f in unmatched if not f)} files ===")
|
||||
other_dirs = {}
|
||||
for rel, is_flagged in unmatched:
|
||||
if not is_flagged:
|
||||
d = os.path.dirname(rel)
|
||||
other_dirs[d] = other_dirs.get(d, 0) + 1
|
||||
for k, v in sorted(other_dirs.items()):
|
||||
print(f" {v:5d} :: {k}")
|
||||
|
||||
if execute:
|
||||
print("\n=== EXECUTING MOVES ===")
|
||||
moved = 0
|
||||
for src, dst in moves:
|
||||
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
||||
if os.path.exists(dst):
|
||||
print(f" SKIP (dst exists): {dst}")
|
||||
continue
|
||||
shutil.move(src, dst)
|
||||
moved += 1
|
||||
print(f"Moved {moved} files.")
|
||||
else:
|
||||
print("\n(dry run only -- pass --execute to actually move files)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Final placement pass for the 8 flagged/reviewed _unsorted items."""
|
||||
import os, shutil, sys
|
||||
|
||||
U = "/data/_unsorted"
|
||||
D = "/data"
|
||||
|
||||
MOVES = [
|
||||
# (src relative to _unsorted, dst absolute under /data)
|
||||
("1 Life admin/10 Management of area 10-19/10.09 Archive for area 10-19/SMS Backups as of 2026-04-07/data",
|
||||
"R Ryan/R.1 Life admin/R.11 Identity & vital records/R.11.55 SMS backups"),
|
||||
("1 Life admin/11 Me & other living things/11.09 Archive for category 11/2025-08-07 AI Deck for Jen.pdf",
|
||||
"J Jen/J.0 Meta/J.00 Meta/J.00.01 Inbox for Jen/2025-08-07 AI Deck for Jen.pdf"),
|
||||
("1 Life admin/11 Me & other living things/11.43 My family/Moms Anandi Book & Screenplay/America's Niece - Proposal.pdf",
|
||||
"F Family/F.5 People/F.50 Family/F.50.20 Pandyas/F.50.21 Nandini Pandya/America's Niece - Proposal.pdf"),
|
||||
("1 Life admin/11 Me & other living things/11.71 My career sales pitch/Ryan_Pandya_Resume___2024_10-1.pdf",
|
||||
"R Ryan/R.2 Career/R.20 Curiosity/R.20.02 Resume/Ryan_Pandya_Resume___2024_10-1.pdf"),
|
||||
("1 Life admin/11 Me & other living things/11.71 My career sales pitch/Ryan_Pandya_Resume.pdf",
|
||||
"R Ryan/R.2 Career/R.20 Curiosity/R.20.02 Resume/Ryan_Pandya_Resume.pdf"),
|
||||
("1 Life admin/12 Home, house, and transport/12.01 Inbox for category 12/2021-03-08 - 6 Dorothy sheet from Jason.pdf",
|
||||
"F Family/F.1 Home, household & finances/F.11 Home & property/F.11.09 Old Homes/2021-2026 Dorothy/2021-03-08 - 6 Dorothy sheet from Jason.pdf"),
|
||||
("1 Life admin/13 Money earned, spent, saved, & owed/13.01 Inbox for category 13/Voided check.jpg",
|
||||
"R Ryan/R.1 Life admin/R.13 Personal finance/R.13.01 Inbox/Voided check.jpg"),
|
||||
("1 Life admin/13 Money earned, spent, saved, & owed/13.41 Bank accounts/Ryan Wells Fargo direct deposit instructions.pdf",
|
||||
"R Ryan/R.1 Life admin/R.13 Personal finance/R.13.01 Inbox/Ryan Wells Fargo direct deposit instructions.pdf"),
|
||||
("1 Life admin/14 My digital life/14.14 My data storage & backups/Google backup key.pdf",
|
||||
"R Ryan/R.1 Life admin/R.14 My digital life/R.14.01 Inbox/Google backup key.pdf"),
|
||||
("3 Hobbies & interests/30 Management for Hobbies/30.01 Inbox for Hobbies/elixirbook.pdf",
|
||||
"R Ryan/R.3 Hobbies & interests/R.35 Computers/R.35.21 Computer & programming language textbooks/elixirbook.pdf"),
|
||||
]
|
||||
|
||||
execute = "--execute" in sys.argv
|
||||
for rel_src, rel_dst in MOVES:
|
||||
src = os.path.join(U, rel_src)
|
||||
dst = os.path.join(D, rel_dst)
|
||||
exists = os.path.exists(src)
|
||||
print(f"{'OK ' if exists else 'MISS'} {src} -> {dst}")
|
||||
if execute and exists:
|
||||
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
||||
shutil.move(src, dst)
|
||||
|
||||
if execute:
|
||||
print("Done.")
|
||||
else:
|
||||
print("\n(dry run -- pass --execute)")
|
||||
@@ -10,12 +10,18 @@ Subcommands:
|
||||
vault search <query> find IDs/notes by code, title, or content
|
||||
vault show <ID> show an ID's note + its /data home + blob count
|
||||
vault create <ID> "<title>" create ID note (folder+folder-note) in /vault AND /data
|
||||
vault move <oldID> <newID> renumber: rename note+folder, rewrite backlinks, move /data dir
|
||||
vault move <oldCode> <newCode> renumber/promote: area, category, or ID code, either side (e.g.
|
||||
cross-level "vault move F.50.09 F.59" promotes an ID to a
|
||||
standalone category); rewrites backlinks, moves /data dir
|
||||
vault rename <ID> "<title>" retitle in place, rewrite backlinks
|
||||
vault sync-data reconcile /data skeleton to match manifest (create missing dirs)
|
||||
vault manifest regenerate the topology manifest note
|
||||
vault report [--pretty] deterministic Librarian report (delegates to vault-librarian.py)
|
||||
vault analyze spawn a Hermes Librarian pass (ad-hoc; uses the cron prompt)
|
||||
vault analyze [--haiku|--sonnet|--opus] [-m MODEL] [--provider PROVIDER]
|
||||
spawn a Hermes Librarian pass. DEFAULT: Opus (smart —
|
||||
draws the auto-fix/flag line correctly, ~$0.14/run so cheap
|
||||
enough for nightly). --sonnet / --haiku step DOWN for
|
||||
cheaper/faster passes; -m/--provider override entirely.
|
||||
"""
|
||||
import os, re, sys, subprocess
|
||||
|
||||
@@ -28,6 +34,19 @@ HERMES = "/run/current-system/sw/bin/hermes"
|
||||
LIBRARIAN = os.path.join(_SELF_DIR, "vault-librarian.py")
|
||||
PROMPT = os.path.join(_SELF_DIR, "vault-librarian-prompt.md")
|
||||
|
||||
# `vault analyze` model tiers. Default is Opus — smart enough to draw the auto-fix/
|
||||
# flag line correctly, and ~$0.14/run so cheap enough for a nightly pass. --sonnet /
|
||||
# --haiku step DOWN. IDs verified against this host's Hermes config + Bedrock provisioning.
|
||||
ANALYZE_TIERS = {
|
||||
"haiku": "us.anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
"sonnet": "global.anthropic.claude-sonnet-5",
|
||||
"opus": "global.anthropic.claude-opus-4-8",
|
||||
}
|
||||
# Only the haiku (us.anthropic.*) tier is known to route via an explicit `bedrock`
|
||||
# provider. The global.anthropic.* IDs use the profile's default provider unless the
|
||||
# caller passes --provider, to avoid a provider/model-registration mismatch.
|
||||
ANALYZE_TIER_PROVIDER = {"haiku": "bedrock"}
|
||||
|
||||
WIKILINK_RE = re.compile(r'(!?\[\[)([^\]#|]+)((?:#[^\]|]+)?(?:\|[^\]]+)?)(\]\])')
|
||||
|
||||
|
||||
@@ -131,7 +150,45 @@ def main():
|
||||
sys.exit(subprocess.run([sys.executable, LIBRARIAN] + argv).returncode)
|
||||
elif cmd == "analyze":
|
||||
prompt = open(PROMPT, encoding="utf-8").read()
|
||||
subprocess.run([HERMES, "-z", prompt])
|
||||
# Model tier: default Opus (smart — draws the auto-fix/flag line correctly,
|
||||
# ~$0.14/run so cheap enough to run nightly). --haiku/--sonnet step DOWN.
|
||||
# Raw -m/--provider override the tier entirely (escape hatch for future models
|
||||
# or non-Bedrock providers). Tier flags are last-wins if combined.
|
||||
tier = "opus"
|
||||
if "--sonnet" in argv:
|
||||
tier = "sonnet"
|
||||
if "--haiku" in argv:
|
||||
tier = "haiku"
|
||||
if "--opus" in argv:
|
||||
tier = "opus"
|
||||
model = ANALYZE_TIERS[tier]
|
||||
provider = ANALYZE_TIER_PROVIDER.get(tier) # None for sonnet/opus (use profile default)
|
||||
if "-m" in argv:
|
||||
i = argv.index("-m")
|
||||
if i + 1 >= len(argv):
|
||||
die("-m requires a model name")
|
||||
model = argv[i + 1]
|
||||
tier = "custom"
|
||||
provider = None # custom model: don't force a provider unless asked
|
||||
if "--provider" in argv:
|
||||
i = argv.index("--provider")
|
||||
if i + 1 >= len(argv):
|
||||
die("--provider requires a provider name")
|
||||
provider = argv[i + 1]
|
||||
hcmd = [HERMES, "-z", prompt, "-m", model]
|
||||
if provider:
|
||||
hcmd += ["--provider", provider]
|
||||
# Banner: headless oneshot buffers output, so show it's alive + what it costs.
|
||||
print(f"→ spawning Hermes Librarian pass [tier={tier} provider={provider or 'profile-default'}]",
|
||||
file=sys.stderr)
|
||||
print(f" model: {model}", file=sys.stderr)
|
||||
print(" output is buffered; a full pass can take 1–3 min before you see anything.",
|
||||
file=sys.stderr)
|
||||
if tier == "opus":
|
||||
print(" step DOWN: vault analyze --sonnet (or --haiku) for cheaper/faster passes",
|
||||
file=sys.stderr)
|
||||
sys.stderr.flush()
|
||||
subprocess.run(hcmd)
|
||||
else:
|
||||
die(f"unknown command '{cmd}'")
|
||||
|
||||
|
||||
+32
-10
@@ -6,27 +6,30 @@ Your job: keep the vault coherent, accurate, up to date, and ontologically sound
|
||||
2. The vault (JDex/map) lives at `/vault`; blob territory at `/data`. Notes are the map,
|
||||
files are the territory, joined by ID number — NEVER copy notes into /data.
|
||||
3. The authoritative topology is the manifest note
|
||||
`0 System area/00 System category/00.05 Vault topology (machine-readable)`.
|
||||
4. All structural operations go through the `vault` CLI at /var/lib/hermes/vault
|
||||
(clobber-safe, dry-run by default, --execute to act, operates on BOTH trees + manifest).
|
||||
`0 System area/05 AI Agent/05 AI Agent.md`.
|
||||
4. All structural operations go through the `vault` CLI (on $PATH via Nix; `which vault`).
|
||||
Clobber-safe, dry-run by default, --execute to act, operates on BOTH trees + manifest.
|
||||
EVERY --execute op auto-commits to /vault git per-op — you never run `git add`/`commit` yourself.
|
||||
|
||||
## Step 1 — get the deterministic report ($0, no judgment)
|
||||
Run: python3 /var/lib/hermes/vault-librarian.py --pretty
|
||||
Run: vault report --pretty
|
||||
This gives you: broken wikilinks (introduced vs pre-existing), cross-tree topology diff,
|
||||
JD structural violations (categories that should be IDs, missing .00/folder-notes,
|
||||
extend-notation `NN+` leaves that need real IDs, >2-level nesting), to-do harvest, and
|
||||
category-health signals. Read it fully before deciding anything.
|
||||
|
||||
## Step 2 — AUTO-FIX (silent, deterministic, reversible)
|
||||
Only do these when the correct action is UNAMBIGUOUS. Use the `vault` CLI, then git-commit.
|
||||
Only do these when the correct action is UNAMBIGUOUS. Use the `vault` CLI with --execute;
|
||||
it commits each op for you (undo = git revert). Do NOT run `git add -A` or `git commit`
|
||||
yourself — that would sweep up Ryan's uncommitted journal/Obsidian edits.
|
||||
- Scaffold missing /data skeleton dirs so every vault ID has a blob home (`vault sync-data --execute`).
|
||||
- Fix a broken wikilink when EXACTLY ONE existing note obviously matches the stale target.
|
||||
- Create a missing `.00` management note or missing folder-note (empty is fine).
|
||||
- Fix a broken wikilink when EXACTLY ONE existing note obviously matches the stale target
|
||||
(use `vault rename`/`vault move` where it applies; otherwise edit the one link via Obsidian MCP).
|
||||
- Create a missing `.00` management note or missing folder-note (`vault create --execute`).
|
||||
- Check off a to-do checkbox that is verifiably complete (linked artifact exists / the note
|
||||
itself states it's done). When in doubt, do NOT check it — flag instead.
|
||||
Commit auto-fixes: cd /vault && git add -A && git commit -m "Librarian: <summary>".
|
||||
|
||||
## Step 3 — FLAG (post to this Matrix room, do NOT act)
|
||||
## Step 3 — FLAG (write to the Inbox, then message this Matrix room)
|
||||
Anything ambiguous or structural where the right answer needs Ryan's judgment:
|
||||
- An ID mis-placed under the wrong category (e.g. journalling under an Identity category),
|
||||
or overlaps/clashes between IDs.
|
||||
@@ -38,7 +41,26 @@ Anything ambiguous or structural where the right answer needs Ryan's judgment:
|
||||
- Any single-home violation, sprawl nearing the 10-category / 10-ID limits, or note that
|
||||
feels ontologically mis-filed.
|
||||
|
||||
NOTE: the `0 System area/09 Archive of pre-RJFI tree` subtree is a SANCTIONED garbage dump
|
||||
HOW to flag (two places, both required):
|
||||
1. APPEND a dated section to the Inbox note (via Obsidian MCP, append mode)
|
||||
`0 System area/01 System Inbox/01 System Inbox.md`
|
||||
Header: `## Librarian flags — YYYY-MM-DD`. Under it, one `### N. <short title>` per flag with
|
||||
the issue, your recommended resolution, and the exact `vault` command you'd run if approved.
|
||||
Then commit ONLY that one file (safe even if Ryan's tree is otherwise dirty):
|
||||
cd /vault && git add "0 System area/01 System Inbox/01 System Inbox.md" \
|
||||
&& git commit -m "Librarian: flags for YYYY-MM-DD"
|
||||
(This single-file add is the ONLY git command you run by hand — everything else auto-commits.
|
||||
Never `git add -A`.)
|
||||
2. Your FINAL RESPONSE is auto-delivered to this Matrix room — make it the mobile-friendly
|
||||
summary (see output contract). The Inbox note holds the detail; the message is the nudge.
|
||||
|
||||
## Step 4 — PUSH (so Obsidian sync + Ryan actually see the changes)
|
||||
Local commits are invisible to Ryan until pushed. As the LAST action, push the vault:
|
||||
cd /vault && git push origin master
|
||||
(Auto-fix ops committed per-op and the Inbox flag commit all get pushed together. If the
|
||||
push fails, say so in your final message — do NOT claim changes landed.)
|
||||
|
||||
NOTE: the `0 System area/09 System Archive` subtree is a SANCTIONED garbage dump
|
||||
(Ryan's ruling). NEVER flag, fix, or scan inside it — its broken links, stray files, and
|
||||
odd structure are all intentional/allowed. The tooling already excludes it.
|
||||
|
||||
|
||||
+4
-9
@@ -61,15 +61,10 @@ def check_structure():
|
||||
for fn in fns:
|
||||
if vl.EXTEND_RE.match(fn):
|
||||
out["extend_notation"].append(os.path.relpath(os.path.join(dp, fn), vl.VAULT))
|
||||
# (b) missing .00 per category + (c) missing folder-note per structural folder
|
||||
for code, rel in s["cats"].items():
|
||||
catf = os.path.join(vl.VAULT, rel)
|
||||
has_dot00 = any(vl.id_code(d) == code + ".00"
|
||||
for d in os.listdir(catf) if os.path.isdir(os.path.join(catf, d)))
|
||||
# a .00 may also be a bare note; check for any NN.00 leaf
|
||||
if not has_dot00:
|
||||
out["missing_dot00"].append(code)
|
||||
# folder-notes: every area/cat/id folder should contain <name>.md
|
||||
# (b) .00 category-management notes are NOT a vault-wide requirement (ruling 2026-07-22:
|
||||
# category-level "about this category" info can live in the category's own SYS.AC note;
|
||||
# .00 is only added ad-hoc when a category needs its own inbox/archive/store). Not checked.
|
||||
# (c) missing folder-note per structural folder
|
||||
for level in ("areas", "cats", "ids"):
|
||||
for code, rel in s[level].items():
|
||||
f = os.path.join(vl.VAULT, rel)
|
||||
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/bin/sh
|
||||
# vault-sync-daemon-notify.sh — fired unconditionally by ExecStopPost= on
|
||||
# EVERY exit of vault-sync-daemon.service (clean stop, restart, crash, kill,
|
||||
# OOM). Distinguishes clean vs. unexpected via systemd's own $SERVICE_RESULT
|
||||
# so routine `systemctl stop/restart` doesn't page like a real failure would.
|
||||
#
|
||||
# systemd passes SERVICE_RESULT / EXIT_CODE / EXIT_STATUS as env vars to
|
||||
# ExecStopPost= (documented in systemd.exec(5) "Environment Variables Set for
|
||||
# Executed Processes" -> systemd.service(5) "Process exit codes").
|
||||
TOKEN_PATH="/home/hermes/.hermes/secrets/ntfy-vault-token"
|
||||
NTFY_URL="https://ntfy.pandu.ski/vault"
|
||||
|
||||
TOKEN=$(cat "$TOKEN_PATH" 2>/dev/null) || exit 0
|
||||
[ -n "$TOKEN" ] || exit 0
|
||||
|
||||
RESULT="${SERVICE_RESULT:-unknown}"
|
||||
HOST=$(hostname)
|
||||
NOW=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
|
||||
|
||||
if [ "$RESULT" = "success" ]; then
|
||||
TITLE="Vault sync daemon stopped"
|
||||
BODY="Clean stop on $HOST at $NOW (manual stop/restart)."
|
||||
TAGS="octagonal_sign"
|
||||
PRIORITY="low"
|
||||
else
|
||||
TITLE="Vault sync daemon DIED"
|
||||
BODY="Unexpected exit on $HOST at $NOW. result=$RESULT exit_code=${EXIT_CODE:-?} exit_status=${EXIT_STATUS:-?}. Restart=always will retry in 5s. Check: journalctl -u vault-sync-daemon -n 50"
|
||||
TAGS="skull,rotating_light"
|
||||
PRIORITY="urgent"
|
||||
fi
|
||||
|
||||
curl -fsS --max-time 5 -X POST \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Title: $TITLE" -H "Tags: $TAGS" -H "Priority: $PRIORITY" \
|
||||
-d "$BODY" "$NTFY_URL" >/dev/null 2>&1 || true
|
||||
+2
-2
@@ -99,7 +99,7 @@ def walk_structure(root, include_meta=False):
|
||||
return {"systems": systems, "areas": areas, "cats": cats, "ids": ids}
|
||||
|
||||
|
||||
ARCHIVE_SUBPATH = os.path.join(SYSTEM_META, "09 Archive of pre-RJFI tree")
|
||||
ARCHIVE_SUBPATH = os.path.join(SYSTEM_META, "09 System Archive")
|
||||
|
||||
|
||||
def is_archived(relpath):
|
||||
@@ -237,7 +237,7 @@ def topology_diff(vault_root=VAULT, data_root=DATA):
|
||||
|
||||
|
||||
# ---- topology manifest (the 00-area source-of-truth note) ----
|
||||
MANIFEST_REL = "0 System area/00 System category/00.05 Vault topology (machine-readable)/00.05 Vault topology (machine-readable).md"
|
||||
MANIFEST_REL = "0 System area/05 AI Agent/05 AI Agent.md"
|
||||
MANIFEST_BEGIN = "<!-- VAULT-TOPOLOGY-JSON:BEGIN -->"
|
||||
MANIFEST_END = "<!-- VAULT-TOPOLOGY-JSON:END -->"
|
||||
|
||||
|
||||
+121
-10
@@ -39,6 +39,57 @@ def _find_cat_folder(root, catcode):
|
||||
return None
|
||||
|
||||
|
||||
# --- code-kind helpers for `move`, which supports ID, category, AND area
|
||||
# codes as either side — including cross-level promotion/demotion (e.g.
|
||||
# "F.50.09" -> "F.59": an ID promoted to a standalone category). ---
|
||||
ID_CODE_RE = re.compile(r'^[RJFI]\.\d\d\.\d\d$')
|
||||
CAT_CODE_RE = re.compile(r'^[RJFI]\.\d\d$')
|
||||
AREA_CODE_RE = re.compile(r'^[RJFI]\.\d$')
|
||||
|
||||
|
||||
def _code_kind(code):
|
||||
"""'id' / 'cat' / 'area' / None (not a recognized code shape)."""
|
||||
if ID_CODE_RE.match(code):
|
||||
return "id"
|
||||
if CAT_CODE_RE.match(code):
|
||||
return "cat"
|
||||
if AREA_CODE_RE.match(code):
|
||||
return "area"
|
||||
return None
|
||||
|
||||
|
||||
def _find_folder_by_code(root, code, kind):
|
||||
code_fn = {"id": vl.id_code, "cat": vl.cat_code, "area": vl.area_code}[kind]
|
||||
for dp, dns, fns in os.walk(root):
|
||||
dns[:] = [d for d in dns if d not in vl.SKIP_DIRS]
|
||||
if code_fn(os.path.basename(dp)) == code:
|
||||
return dp
|
||||
return None
|
||||
|
||||
|
||||
def _nested_jd_children(folder):
|
||||
"""
|
||||
List any descendant folder (relative path) whose basename is itself
|
||||
JD-coded (area/category/ID shaped). Moving/renumbering a category or
|
||||
area whose children's own folder names embed the OLD number (e.g.
|
||||
"F.50.11 Ryan" inside category "F.50 Family") would orphan those
|
||||
children's codes — that's a bulk-renumber operation, not a simple
|
||||
move, so `move` refuses rather than silently leaving stale numbers.
|
||||
An ID-kind source is always safe here since IDs never have nested
|
||||
JD-coded children by convention; the check is generic and simply
|
||||
finds nothing in that case.
|
||||
"""
|
||||
found = []
|
||||
for dp, dns, fns in os.walk(folder):
|
||||
dns[:] = [d for d in dns if d not in vl.SKIP_DIRS]
|
||||
if dp == folder:
|
||||
continue
|
||||
base = os.path.basename(dp)
|
||||
if vl.ID_RE.match(base) or vl.CAT_RE.match(base) or vl.AREA_RE.match(base):
|
||||
found.append(os.path.relpath(dp, folder))
|
||||
return found
|
||||
|
||||
|
||||
def _rewrite_links(old_stem, new_stem, execute):
|
||||
"""Rewrite [[old_stem]] -> [[new_stem]] vault-wide. Returns count."""
|
||||
n = 0
|
||||
@@ -101,24 +152,82 @@ def op_create(argv, execute):
|
||||
|
||||
|
||||
def op_move(argv, execute):
|
||||
"""
|
||||
Renumber/move an area, category, or ID folder to a new code — including
|
||||
cross-level promotion/demotion (e.g. an ID promoted to a standalone
|
||||
category: `vault move F.50.09 F.59`). Same-level renumbers (ID->ID,
|
||||
cat->cat, area->area) work as before; the source and destination no
|
||||
longer have to be the same kind.
|
||||
|
||||
Refuses (clobber-safe) if:
|
||||
- the destination code already exists anywhere in that system
|
||||
- the source has JD-coded descendants (moving a category/area that
|
||||
still has numbered children would orphan their codes — that's a
|
||||
bulk-renumber, a different and much riskier operation, not this one)
|
||||
- moving cross-system (letter must match; there's no valid target
|
||||
parent on a different system)
|
||||
"""
|
||||
if len(argv) < 2:
|
||||
_die("usage: vault move <oldID> <newID>")
|
||||
_die("usage: vault move <oldCode> <newCode> (ID, category, or area code; can differ in kind)")
|
||||
old, new = argv[0], argv[1]
|
||||
old_kind = _code_kind(old)
|
||||
new_kind = _code_kind(new)
|
||||
if not old_kind:
|
||||
_die(f"'{old}' is not a recognized area/category/ID code (expected SYS.N, SYS.NN, or SYS.NN.NN)")
|
||||
if not new_kind:
|
||||
_die(f"'{new}' is not a recognized area/category/ID code (expected SYS.N, SYS.NN, or SYS.NN.NN)")
|
||||
if old[0] != new[0]:
|
||||
_die(f"cross-system move not supported: {old} is system '{old[0]}', {new} is system '{new[0]}'")
|
||||
|
||||
root = _sys_root(old[0])
|
||||
oldf = _find_id_folder(root, old)
|
||||
oldf = _find_folder_by_code(root, old, old_kind)
|
||||
if not oldf:
|
||||
_die(f"source ID {old} not found")
|
||||
_die(f"source {old_kind} {old} not found")
|
||||
|
||||
newroot = _sys_root(new[0])
|
||||
if _find_id_folder(newroot, new):
|
||||
_die(f"CLOBBER: destination ID {new} already exists")
|
||||
newcat = new[:new.rindex(".")]
|
||||
newcatf = _find_cat_folder(newroot, newcat)
|
||||
if not newcatf:
|
||||
_die(f"destination category {newcat} does not exist")
|
||||
if _find_folder_by_code(newroot, new, new_kind):
|
||||
_die(f"CLOBBER: destination {new_kind} {new} already exists")
|
||||
|
||||
nested = _nested_jd_children(oldf)
|
||||
if nested:
|
||||
kind_word = {"id": "ID", "cat": "category", "area": "area"}[old_kind]
|
||||
_die(
|
||||
f"{old} ({kind_word}) has {len(nested)} JD-coded child folder(s) "
|
||||
f"(e.g. {nested[0]}) — moving it would orphan their codes. "
|
||||
f"This needs a bulk renumber, not `move`. Refusing."
|
||||
)
|
||||
|
||||
if old_kind != new_kind:
|
||||
print(f"NOTE: cross-level move — {old} is a {old_kind}, {new} is a {new_kind}.")
|
||||
|
||||
# Destination's PARENT folder must exist. For a same-kind move this is
|
||||
# "the category/area that will contain the ID/category". For a
|
||||
# cross-level promotion (e.g. id -> cat), the parent is one level up
|
||||
# from what `new`'s own kind would normally sit under — i.e. new's
|
||||
# area, found the normal way regardless of what kind old was.
|
||||
# NB: parent code can't be derived by slicing at the last '.' — a
|
||||
# category code like "F.58" has only ONE dot, so that would yield "F"
|
||||
# (a system letter) instead of "F.5" (the area it belongs to). Derive
|
||||
# it from the numeric digits directly instead.
|
||||
if new_kind == "area":
|
||||
newparentf = newroot # area's parent is the system root itself
|
||||
else:
|
||||
if new_kind == "id":
|
||||
parent_kind = "cat"
|
||||
parent_code = new[: new.rindex(".")] # "F.58.03" -> "F.58"
|
||||
else: # new_kind == "cat"
|
||||
parent_kind = "area"
|
||||
catnum = new.split(".", 1)[1] # "F.58" -> "58"
|
||||
parent_code = f"{new[0]}.{catnum[0]}" # -> "F.5"
|
||||
found_parent = _find_folder_by_code(newroot, parent_code, parent_kind)
|
||||
if not found_parent:
|
||||
_die(f"destination parent {parent_kind} {parent_code} does not exist — create it first")
|
||||
newparentf = found_parent
|
||||
|
||||
old_base = os.path.basename(oldf)
|
||||
title = old_base.split(" ", 1)[1] if " " in old_base else ""
|
||||
new_base = f"{new} {title}".strip()
|
||||
new_folder = os.path.join(newcatf, new_base)
|
||||
new_folder = os.path.join(newparentf, new_base)
|
||||
old_stem, new_stem = old_base, new_base
|
||||
old_drel = os.path.relpath(oldf, vl.VAULT)
|
||||
new_drel = os.path.relpath(new_folder, vl.VAULT)
|
||||
@@ -133,6 +242,7 @@ def op_move(argv, execute):
|
||||
oldnote = os.path.join(oldf, old_base + ".md")
|
||||
if os.path.exists(oldnote):
|
||||
os.rename(oldnote, os.path.join(oldf, new_base + ".md"))
|
||||
os.makedirs(newparentf, exist_ok=True)
|
||||
os.rename(oldf, new_folder)
|
||||
olddata = os.path.join(vl.DATA, old_drel)
|
||||
if os.path.isdir(olddata):
|
||||
@@ -198,6 +308,7 @@ def op_sync_data(argv, execute):
|
||||
if len(made) > 40:
|
||||
print(f" ... and {len(made)-40} more")
|
||||
if execute:
|
||||
_git_commit(f"vault sync-data ({len(made)} /data dir(s))", execute)
|
||||
print("EXECUTED.")
|
||||
else:
|
||||
print("DRY-RUN — nothing changed. Add --execute.")
|
||||
|
||||
@@ -0,0 +1,945 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
vault_sync_daemon.py — long-running inotify (watchdog) daemon that keeps
|
||||
/vault (JDex notes, git-backed) and /data (blob storage, Duplicati-backed)
|
||||
structurally mirrored, in BOTH directions, for changes made by ANY tool
|
||||
(Obsidian, SMB client, manual mv, not just the `vault` CLI).
|
||||
|
||||
Rules (confirmed with Ryan 2026-09-15):
|
||||
|
||||
FORWARD (/vault -> /data):
|
||||
- any non-`.md` file created or moved INTO /vault (a binary dropped or
|
||||
saved there by mistake -- /vault is notes-only by design) -> relocate
|
||||
it to the matching path under /data instead of leaving it in /vault.
|
||||
- rename/move of an ID folder in /vault -> mirror the rename in /data.
|
||||
- delete of an ID folder in /vault -> do NOT delete /data's copy.
|
||||
Instead: move the /data folder into the NEAREST existing `.09` archive
|
||||
folder up the tree (category-level first, then area/system; create a
|
||||
category-level `.09 Archive` in /data if none exists at any level yet),
|
||||
and append a note to the nearest `.09` archive's /vault folder-note
|
||||
markdown recording what was archived, when, and where.
|
||||
|
||||
REVERSE (/data -> /vault):
|
||||
- a new folder appears in /data whose basename matches the `SYS.AC.ID `
|
||||
Johnny Decimal pattern, with NO /vault counterpart yet (e.g. dropped in
|
||||
over SMB) -> auto-create a minimal /vault stub folder + folder-note
|
||||
with just `aliases: [<title with ID stripped>]` frontmatter (no
|
||||
"needs review" banner, no inbox #fix-me flagging — confirmed with
|
||||
Ryan 2026-09-22: keep it quiet, just the empty note + alias).
|
||||
- Folders inside any `.09`-pattern archive folder are never auto-stubbed
|
||||
(archives are meant to hold retired stuff, not grow new tracked IDs) —
|
||||
this also breaks the feedback loop from the forward-delete archive move.
|
||||
- Generic non-ID subfolders (e.g. "Documents/" inside an existing ID
|
||||
folder) are ignored entirely — normal blob organization, not JD structure.
|
||||
|
||||
Idempotent by construction: every action checks target state first, so a
|
||||
duplicate/replayed event from the daemon's own writes is a safe no-op.
|
||||
|
||||
Run: python3 vault_sync_daemon.py (foreground; systemd unit wraps this)
|
||||
Logs: /home/hermes/.hermes/logs/vault-sync-daemon.log
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import shutil
|
||||
import subprocess
|
||||
import datetime
|
||||
import threading
|
||||
import queue
|
||||
|
||||
_SELF_DIR = os.path.dirname(os.path.realpath(__file__))
|
||||
sys.path.insert(0, _SELF_DIR)
|
||||
import vault_lib as vl
|
||||
|
||||
from watchdog.observers import Observer
|
||||
from watchdog.events import FileSystemEventHandler, DirCreatedEvent, DirMovedEvent, DirDeletedEvent
|
||||
|
||||
LOG_PATH = "/home/hermes/.hermes/logs/vault-sync-daemon.log"
|
||||
os.makedirs(os.path.dirname(LOG_PATH), exist_ok=True)
|
||||
|
||||
NTFY_TOKEN_PATH = "/home/hermes/.hermes/secrets/ntfy-vault-token"
|
||||
NTFY_URL = "https://ntfy.pandu.ski/vault"
|
||||
|
||||
|
||||
def ntfy_post(title, message, tags="package", priority="default"):
|
||||
"""Fire-and-forget ntfy notification. Never raises — a notify failure
|
||||
must never block or crash the daemon's actual sync work."""
|
||||
try:
|
||||
token = open(NTFY_TOKEN_PATH, encoding="utf-8").read().strip()
|
||||
except Exception as e:
|
||||
log(f"ntfy_post: could not read token: {e}")
|
||||
return
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["curl", "-fsS", "--max-time", "5", "-X", "POST",
|
||||
"-H", f"Authorization: Bearer {token}",
|
||||
"-H", f"Title: {title}",
|
||||
"-H", f"Tags: {tags}",
|
||||
"-H", f"Priority: {priority}",
|
||||
"-d", message,
|
||||
NTFY_URL],
|
||||
check=False, capture_output=True, timeout=8, text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
log(f"ntfy_post: curl exited {result.returncode}: {result.stderr.strip()}")
|
||||
except Exception as e:
|
||||
log(f"ntfy_post failed: {e}")
|
||||
|
||||
VAULT = vl.VAULT
|
||||
DATA = vl.DATA
|
||||
SKIP_DIRS = vl.SKIP_DIRS
|
||||
DATA_EXEMPT = vl.DATA_EXEMPT
|
||||
|
||||
ID_RE = vl.ID_RE
|
||||
CAT_RE = vl.CAT_RE
|
||||
ARCHIVE_ID_RE = re.compile(r'^([RJFI])\.(\d\d)\.09 ') # e.g. "F.25.09 Archive"
|
||||
ARCHIVE_CAT_RE = re.compile(r'^([RJFI])\.(\d\d) ') # category folder, to find/create .09 inside
|
||||
|
||||
# Obsidian's "New folder" action does `mkdir Untitled` (or "Untitled 1", "2",
|
||||
# etc. if that name's taken) the INSTANT you click it, before you've typed a
|
||||
# real name -- it is never a real OS rename() afterward, just a second
|
||||
# mkdir-with-real-name once you hit enter (confirmed with Ryan 2026-09-22).
|
||||
# The "Untitled" placeholder is always transient and must never be mirrored
|
||||
# to /data: doing so left permanent orphan junk there (the real delete event
|
||||
# for "Untitled" gets silently dropped by handle_vault_deleted, which only
|
||||
# tracks ID-shaped names) and produced two separate "create" ntfy pings
|
||||
# instead of the one the user actually cares about.
|
||||
UNTITLED_RE = re.compile(r'^Untitled(\s\d+)?$')
|
||||
|
||||
# Journal entries: /vault/R Ryan/.../R.16.11 My journal & notes by day/YYYY/MM/YYYY-MM-DD.md
|
||||
# gets an auto-injected `title: MMM D, YYYY` frontmatter property (matches the
|
||||
# format Ryan has used by hand on every entry so far -- full month name, NO
|
||||
# zero-padded day, e.g. "title: September 3, 2026" not "September 03, 2026").
|
||||
JOURNAL_DIR = os.path.join(
|
||||
VAULT, "R Ryan", "R.1 Life admin", "R.16 Journal & personal writing",
|
||||
"R.16.11 My journal & notes by day",
|
||||
)
|
||||
JOURNAL_FILENAME_RE = re.compile(r'^(\d{4})-(\d{2})-(\d{2})\.md$')
|
||||
_MONTH_NAMES = ["January", "February", "March", "April", "May", "June", "July",
|
||||
"August", "September", "October", "November", "December"]
|
||||
|
||||
_lock = threading.Lock()
|
||||
_recent_events = {} # (kind, path) -> timestamp, for debouncing duplicate inotify events
|
||||
_DEBOUNCE_SECONDS = 3.0
|
||||
|
||||
# Watchdog's inotify cookie-pairing (MOVED_FROM + MOVED_TO -> one on_moved)
|
||||
# is best-effort and confirmed to fail even on a single, unloaded, plain
|
||||
# folder rename (see handle_vault_deleted docstring) -- not just under
|
||||
# burst load like the earlier "Untitled" bug. To recover the rename intent
|
||||
# from an unpaired delete+create pair, a vault-side ID deletion is held
|
||||
# here (keyed by JD ID code, e.g. "F.13.13") for RENAME_GRACE_SECONDS
|
||||
# before it's actually archived, giving a same-ID create a chance to
|
||||
# claim it as a rename instead.
|
||||
_pending_id_deletes = {}
|
||||
_pending_deletes_lock = threading.Lock()
|
||||
RENAME_GRACE_SECONDS = 15.0
|
||||
|
||||
|
||||
def _debounced(kind, path):
|
||||
"""
|
||||
Return True (and record) if this (kind, path) event was NOT seen in the
|
||||
last _DEBOUNCE_SECONDS — inotify/watchdog can emit duplicate CREATE
|
||||
events for a single mkdir, so every handler must be idempotent AND
|
||||
debounced to avoid double-processing (e.g. flagging the same new /data
|
||||
folder in two different inbox notes).
|
||||
"""
|
||||
now = time.time()
|
||||
key = (kind, path)
|
||||
last = _recent_events.get(key)
|
||||
_recent_events[key] = now
|
||||
# opportunistic cleanup so this dict never grows unbounded
|
||||
if len(_recent_events) > 2000:
|
||||
cutoff = now - _DEBOUNCE_SECONDS
|
||||
for k, v in list(_recent_events.items()):
|
||||
if v < cutoff:
|
||||
del _recent_events[k]
|
||||
return last is None or (now - last) > _DEBOUNCE_SECONDS
|
||||
|
||||
|
||||
def log(msg):
|
||||
line = f"{datetime.datetime.now().isoformat(timespec='seconds')} {msg}"
|
||||
print(line, flush=True)
|
||||
try:
|
||||
with open(LOG_PATH, "a", encoding="utf-8") as f:
|
||||
f.write(line + "\n")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _git_commit(msg, paths=None):
|
||||
"""
|
||||
Commit only the specific paths the daemon itself touched — never a
|
||||
blanket `git add -A`, so a human's concurrent in-progress edits
|
||||
elsewhere in the vault are never swept into an unrelated auto-commit.
|
||||
"""
|
||||
if not paths:
|
||||
return
|
||||
subprocess.run(["git", "-C", VAULT, "add", "--"] + list(paths), check=False)
|
||||
diff = subprocess.run(["git", "-C", VAULT, "diff", "--cached", "--quiet"], check=False)
|
||||
if diff.returncode == 0:
|
||||
return # nothing staged
|
||||
subprocess.run(["git", "-C", VAULT, "commit", "-m", msg], check=False)
|
||||
|
||||
|
||||
def _in_skip(path, root):
|
||||
rel = os.path.relpath(path, root)
|
||||
parts = rel.split(os.sep)
|
||||
return any(p in SKIP_DIRS for p in parts)
|
||||
|
||||
|
||||
def _under_archive(rel_path):
|
||||
"""True if any path component matches the '<SYS.AC>.09 ' archive pattern."""
|
||||
for part in rel_path.split(os.sep):
|
||||
if ARCHIVE_ID_RE.match(part):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _under_data_exempt(rel_path):
|
||||
top = rel_path.split(os.sep)[0]
|
||||
return top in DATA_EXEMPT
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# FORWARD: /vault -> /data
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def handle_vault_created(path):
|
||||
"""
|
||||
A new folder appeared in /vault (e.g. created via Obsidian, not the
|
||||
`vault` CLI). Mirror it: create the matching empty folder in /data.
|
||||
This is the forward-create leg that was missing — only move/delete
|
||||
were wired up originally, so a brand-new folder (like "R.6 Test")
|
||||
never propagated.
|
||||
|
||||
Full 1:1 structural mirror, NOT limited to JD-shaped (AREA/CATEGORY/ID)
|
||||
names (confirmed with Ryan 2026-09-19): once inside an ID's territory
|
||||
Ryan organizes arbitrary sub-structure (e.g. dated trip folders under
|
||||
an ID like "F.21.21 All short trips"), and expects /data to match that
|
||||
structure too, not just the JD skeleton. The only safety gate is that
|
||||
the /data PARENT must already exist — this naturally still excludes
|
||||
"0 System area" (which has no /data root at all, by design) and any
|
||||
other genuinely orphaned branch, without needing a name-shape check.
|
||||
Reverse (/data -> /vault) intentionally keeps its stricter JD-shape-only
|
||||
gate (see handle_data_created) — /data is the hand-sorted blob side
|
||||
with far more incidental subfolders, so stubbing a /vault note for
|
||||
every one of those would be noise; there's no such asymmetry forward.
|
||||
"""
|
||||
if not os.path.isdir(path):
|
||||
return
|
||||
if _in_skip(path, VAULT):
|
||||
return
|
||||
if not _debounced("vault_created", path):
|
||||
return
|
||||
rel = os.path.relpath(path, VAULT)
|
||||
if rel.split(os.sep)[0] == vl.SYSTEM_META:
|
||||
return # "0 System area" never mirrors to /data, by design
|
||||
if UNTITLED_RE.match(os.path.basename(path)):
|
||||
# Obsidian's transient "New folder" placeholder -- never mirror it.
|
||||
# The real create (with the name Ryan actually typed) fires as its
|
||||
# own separate event moments later and mirrors normally, so skipping
|
||||
# this one is enough: no orphan junk in /data, no duplicate ntfy.
|
||||
return
|
||||
base = os.path.basename(path)
|
||||
id_code = vl.id_code(base)
|
||||
if id_code:
|
||||
with _pending_deletes_lock:
|
||||
pending = _pending_id_deletes.pop(id_code, None)
|
||||
if pending is not None:
|
||||
# A vault-side delete of the SAME JD ID landed within the
|
||||
# grace window -- this create is the other half of a rename
|
||||
# that watchdog failed to pair into one on_moved (confirmed
|
||||
# 2026-09-24, see handle_vault_deleted). Recover the rename
|
||||
# instead of doing a normal create-mirror (which would have
|
||||
# produced a fresh empty stub, orphaning whatever was really
|
||||
# in the old-named /data folder).
|
||||
_resolve_as_rename(pending, path, rel)
|
||||
return
|
||||
data_path = os.path.join(DATA, rel)
|
||||
if os.path.isdir(data_path):
|
||||
return # already mirrored (duplicate event or pre-existing)
|
||||
parent = os.path.dirname(data_path)
|
||||
if not os.path.isdir(parent):
|
||||
log(f"SKIP create mirror (parent missing in /data): {rel}")
|
||||
return
|
||||
os.makedirs(data_path, exist_ok=True)
|
||||
log(f"MIRRORED create /data: {rel}")
|
||||
ntfy_post(
|
||||
"Vault sync: new folder",
|
||||
f"Created in /vault, mirrored to /data: {rel}",
|
||||
tags="sparkles",
|
||||
)
|
||||
|
||||
|
||||
def handle_vault_moved(src_path, dest_path):
|
||||
if _in_skip(src_path, VAULT) or _in_skip(dest_path, VAULT):
|
||||
return
|
||||
rel_src = os.path.relpath(src_path, VAULT)
|
||||
rel_dest = os.path.relpath(dest_path, VAULT)
|
||||
data_src = os.path.join(DATA, rel_src)
|
||||
data_dest = os.path.join(DATA, rel_dest)
|
||||
if not os.path.isdir(data_src):
|
||||
# No /data counterpart to rename -- most commonly because the
|
||||
# source was an "Untitled" placeholder we deliberately never
|
||||
# mirrored (see UNTITLED_RE). Don't just drop this silently: the
|
||||
# folder (now under its real name) still needs to exist in /data,
|
||||
# so fall through to a normal create-mirror at the destination.
|
||||
if os.path.exists(data_dest):
|
||||
return # already mirrored somehow, nothing to do
|
||||
parent = os.path.dirname(data_dest)
|
||||
if not os.path.isdir(parent):
|
||||
log(f"SKIP move-as-create mirror (parent missing in /data): {rel_dest}")
|
||||
return
|
||||
os.makedirs(data_dest, exist_ok=True)
|
||||
log(f"MIRRORED create /data (from untitled-rename): {rel_dest}")
|
||||
ntfy_post(
|
||||
"Vault sync: new folder",
|
||||
f"Created in /vault, mirrored to /data: {rel_dest}",
|
||||
tags="sparkles",
|
||||
)
|
||||
return
|
||||
if os.path.exists(data_dest):
|
||||
log(f"SKIP move mirror (dest exists): {rel_src} -> {rel_dest}")
|
||||
return
|
||||
os.makedirs(os.path.dirname(data_dest), exist_ok=True)
|
||||
os.rename(data_src, data_dest)
|
||||
log(f"MIRRORED rename /data: {rel_src} -> {rel_dest}")
|
||||
ntfy_post(
|
||||
"Vault sync: renamed",
|
||||
f"/data: {rel_src} -> {rel_dest}",
|
||||
tags="package",
|
||||
)
|
||||
|
||||
|
||||
def handle_vault_binary_created(path):
|
||||
"""
|
||||
/vault is notes-only by design (see vault-librarian skill). Any
|
||||
non-`.md` file that lands there -- Ryan saving a photo/PDF by hand,
|
||||
an agent writing to the wrong tree, a sync client dropping a stray
|
||||
file -- gets relocated to the matching path under /data, never left
|
||||
in /vault. Idempotent: no-op if the file is already gone (raced with
|
||||
another mover) or the /data destination already exists (name clash --
|
||||
log and leave the vault copy in place rather than silently overwriting
|
||||
or deleting real content).
|
||||
"""
|
||||
if not os.path.isfile(path):
|
||||
return
|
||||
if path.endswith(".md"):
|
||||
return
|
||||
if _in_skip(path, VAULT):
|
||||
return
|
||||
if not _debounced("vault_binary_created", path):
|
||||
return
|
||||
rel = os.path.relpath(path, VAULT)
|
||||
if rel.split(os.sep)[0] == vl.SYSTEM_META:
|
||||
return # "0 System area" has no /data counterpart at all
|
||||
data_path = os.path.join(DATA, rel)
|
||||
if os.path.exists(data_path):
|
||||
log(f"SKIP relocate (already exists in /data, leaving vault copy): {rel}")
|
||||
ntfy_post(
|
||||
"Vault sync: binary needs manual move",
|
||||
f"Non-.md file in /vault couldn't relocate -- /data already "
|
||||
f"has {rel}. Left in place, resolve by hand.",
|
||||
tags="warning",
|
||||
priority="high",
|
||||
)
|
||||
return
|
||||
os.makedirs(os.path.dirname(data_path), exist_ok=True)
|
||||
shutil.move(path, data_path)
|
||||
log(f"RELOCATED non-md file /vault -> /data: {rel}")
|
||||
ntfy_post(
|
||||
"Vault sync: moved binary out of /vault",
|
||||
f"{rel} isn't a note -- moved to /data.",
|
||||
tags="package",
|
||||
)
|
||||
|
||||
|
||||
def handle_journal_file_created(path):
|
||||
"""
|
||||
A new dated journal entry (YYYY-MM-DD.md) appeared under R.16.11 --
|
||||
auto-inject `title: MMM D, YYYY` frontmatter (no zero-padded day),
|
||||
matching the format Ryan has applied by hand to every entry so far.
|
||||
Idempotent: no-op if the file already has a `title:` line in its
|
||||
frontmatter block, or isn't actually under the journal dir / doesn't
|
||||
match the dated-entry filename shape.
|
||||
"""
|
||||
if not os.path.isfile(path):
|
||||
return
|
||||
if not path.endswith(".md"):
|
||||
return
|
||||
try:
|
||||
rel_to_journal = os.path.relpath(path, JOURNAL_DIR)
|
||||
except ValueError:
|
||||
return
|
||||
if rel_to_journal.startswith(".."):
|
||||
return # not actually under the journal dir
|
||||
base = os.path.basename(path)
|
||||
m = JOURNAL_FILENAME_RE.match(base)
|
||||
if not m:
|
||||
return # not a dated entry (e.g. some other note dropped in here)
|
||||
if not _debounced("journal_title", path):
|
||||
return
|
||||
year, month, day = int(m.group(1)), int(m.group(2)), int(m.group(3))
|
||||
month_name = _MONTH_NAMES[month - 1]
|
||||
title_value = f"{month_name} {day}, {year}"
|
||||
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
text = f.read()
|
||||
except Exception as e:
|
||||
log(f"ERROR journal title read failed: {path}: {e}")
|
||||
return
|
||||
|
||||
if text.startswith("---\n"):
|
||||
end = text.find("\n---", 4)
|
||||
if end == -1:
|
||||
log(f"SKIP journal title (malformed frontmatter, no closing ---): {rel_to_journal}")
|
||||
return
|
||||
frontmatter = text[4:end]
|
||||
rest = text[end:] # starts with "\n---"
|
||||
if re.search(r'^title:\s*', frontmatter, re.MULTILINE):
|
||||
return # already has a title, never overwrite a human-set value
|
||||
new_frontmatter = f"title: {title_value}\n" + frontmatter
|
||||
new_text = "---\n" + new_frontmatter + rest
|
||||
else:
|
||||
# no frontmatter block at all yet -- create one
|
||||
new_text = f"---\ntitle: {title_value}\n---\n" + text
|
||||
|
||||
try:
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write(new_text)
|
||||
except Exception as e:
|
||||
log(f"ERROR journal title write failed: {path}: {e}")
|
||||
return
|
||||
log(f"TITLED journal entry: {rel_to_journal} -> \"{title_value}\"")
|
||||
_git_commit(f"vault_sync_daemon: add title to journal entry {base}", [path])
|
||||
|
||||
|
||||
def _category_title(catname):
|
||||
"""'F.22 Fun lists' -> 'Fun lists'"""
|
||||
m = CAT_RE.match(catname)
|
||||
return catname[m.end():].strip() if m else catname
|
||||
|
||||
|
||||
def _find_nearest_archive_dir(rel_path):
|
||||
"""
|
||||
Archive destination is ALWAYS the immediate owning category of the
|
||||
deleted ID (one level up from the ID folder) — never a higher-level
|
||||
ancestor's archive, even if one already exists there. If that category
|
||||
has no '<SYS.AC>.09 ' archive folder yet, create it right there
|
||||
(materializing the missing JD archive slot), matching Ryan's rule:
|
||||
"create missing .01s and .09s when they're missing", not borrow one
|
||||
further up the tree.
|
||||
Returns (archive_data_path, vault_note_path_or_None).
|
||||
vault_note_path is set only if this call just created a NEW archive
|
||||
slot's folder-note (so the caller knows to git-add it).
|
||||
"""
|
||||
parts = rel_path.split(os.sep)
|
||||
if len(parts) < 2:
|
||||
return (DATA, None) # ID sits at top level (shouldn't happen)
|
||||
cat_rel = os.sep.join(parts[:-1])
|
||||
cat_vault = os.path.join(VAULT, cat_rel)
|
||||
cat_data = os.path.join(DATA, cat_rel)
|
||||
catname = os.path.basename(cat_vault)
|
||||
m = CAT_RE.match(catname)
|
||||
if not m or not os.path.isdir(cat_vault):
|
||||
return (cat_data if os.path.isdir(cat_data) else DATA, None)
|
||||
sysletter, catnum = m.group(1), m.group(2)
|
||||
# already has an archive slot in this category?
|
||||
if os.path.isdir(cat_vault):
|
||||
for name in sorted(os.listdir(cat_vault)):
|
||||
if ARCHIVE_ID_RE.match(name):
|
||||
dpath = os.path.join(cat_data, name)
|
||||
os.makedirs(dpath, exist_ok=True)
|
||||
return (dpath, None)
|
||||
# missing — create it in THIS category, not a higher one
|
||||
archive_name = f"{sysletter}.{catnum}.09 Archive for {_category_title(catname)}"
|
||||
vpath = os.path.join(cat_vault, archive_name)
|
||||
vnote = os.path.join(vpath, archive_name + ".md")
|
||||
created_note = None
|
||||
if not os.path.exists(vnote):
|
||||
os.makedirs(vpath, exist_ok=True)
|
||||
with open(vnote, "w", encoding="utf-8") as f:
|
||||
f.write(f"# {archive_name}\n\n"
|
||||
f"Auto-created by vault_sync_daemon on "
|
||||
f"{datetime.datetime.now().isoformat(timespec='seconds')} "
|
||||
f"as the archive destination for a deleted ID in this category.\n")
|
||||
log(f"CREATED category archive slot: {os.path.relpath(vpath, VAULT)}")
|
||||
created_note = vnote
|
||||
archive_path = os.path.join(cat_data, archive_name)
|
||||
os.makedirs(archive_path, exist_ok=True)
|
||||
return (archive_path, created_note)
|
||||
|
||||
|
||||
def _nearest_archive_note(archive_data_path):
|
||||
"""Given an archive dir in /data, return the matching /vault folder-note path, if any."""
|
||||
rel = os.path.relpath(archive_data_path, DATA)
|
||||
vault_dir = os.path.join(VAULT, rel)
|
||||
if not os.path.isdir(vault_dir):
|
||||
return None
|
||||
note = os.path.join(vault_dir, os.path.basename(vault_dir) + ".md")
|
||||
return note if os.path.exists(note) else None
|
||||
|
||||
|
||||
def _prune_deleted_plain_folder(vault_path, rel, data_path):
|
||||
"""
|
||||
A non-ID folder vanished from /vault (typically a sync client applying
|
||||
a rename as delete+create, so the old name's /data mirror is orphaned).
|
||||
Remove the /data mirror only if its whole tree holds no files; if it
|
||||
holds any, leave it and ping instead.
|
||||
"""
|
||||
if rel.split(os.sep)[0] == vl.SYSTEM_META or _under_archive(rel):
|
||||
return
|
||||
if os.path.exists(vault_path) or not os.path.isdir(data_path):
|
||||
return
|
||||
if any(files for _, _, files in os.walk(data_path)):
|
||||
log(f"KEPT /data mirror of deleted vault folder (has files): {rel}")
|
||||
ntfy_post(
|
||||
"Vault sync: deleted folder has blobs",
|
||||
f"Folder removed from /vault but /data/{rel} contains files. Left in place.",
|
||||
tags="warning",
|
||||
)
|
||||
return
|
||||
shutil.rmtree(data_path)
|
||||
log(f"PRUNED empty /data mirror of deleted vault folder: {rel}")
|
||||
|
||||
|
||||
def handle_vault_deleted(path):
|
||||
"""
|
||||
An ID-shaped folder disappeared from /vault. Per design this normally
|
||||
means archive the /data copy (see module docstring) -- but a bare
|
||||
on_deleted this is NOT always a real deletion. Confirmed 2026-09-24:
|
||||
a plain, unloaded, single-folder Obsidian rename (no burst, no stress
|
||||
test -- just "Budget & cash-flow tracking" -> "...tracker") produced
|
||||
an unpaired on_deleted + on_created ~12s apart instead of one
|
||||
on_moved. This is the SAME watchdog cookie-pairing gap as the earlier
|
||||
Untitled-placeholder bug, but this time with no load to blame at all
|
||||
-- so it can happen on any rename, not just bursts.
|
||||
|
||||
Fix: don't archive immediately. Hold the delete in
|
||||
`_pending_id_deletes` keyed by the JD ID code (e.g. "F.13.13") for
|
||||
RENAME_GRACE_SECONDS. If a create for a folder with the SAME id code
|
||||
(different title text is fine -- that's exactly what a rename is)
|
||||
shows up within the window, handle_vault_created finds it in the
|
||||
pending dict and treats the pair as the rename it actually was
|
||||
instead of archiving-then-restubbing. Only after the grace window
|
||||
passes with no matching create does this actually archive.
|
||||
"""
|
||||
if _in_skip(path, VAULT):
|
||||
return
|
||||
if not _debounced("vault_deleted", path):
|
||||
return
|
||||
rel = os.path.relpath(path, VAULT)
|
||||
base = os.path.basename(path)
|
||||
data_path = os.path.join(DATA, rel)
|
||||
if not ID_RE.match(base):
|
||||
_prune_deleted_plain_folder(path, rel, data_path)
|
||||
return
|
||||
if not os.path.isdir(data_path):
|
||||
return # already handled (duplicate event) or no /data counterpart ever existed
|
||||
id_code = vl.id_code(base)
|
||||
if id_code:
|
||||
with _pending_deletes_lock:
|
||||
_pending_id_deletes[id_code] = {"path": path, "rel": rel, "base": base}
|
||||
log(f"PENDING delete (grace window for possible rename pairing): {rel}")
|
||||
timer = threading.Timer(RENAME_GRACE_SECONDS, _finalize_pending_delete, args=(id_code,))
|
||||
timer.daemon = True
|
||||
timer.start()
|
||||
return
|
||||
_archive_deleted_id(path, rel, base)
|
||||
|
||||
|
||||
def _finalize_pending_delete(id_code):
|
||||
"""Timer callback: if nothing claimed this id_code as a rename target
|
||||
within the grace window, it really was just a deletion -- archive now."""
|
||||
with _pending_deletes_lock:
|
||||
entry = _pending_id_deletes.pop(id_code, None)
|
||||
if entry is None:
|
||||
return # a matching create already resolved this as a rename
|
||||
with _lock:
|
||||
_archive_deleted_id(entry["path"], entry["rel"], entry["base"])
|
||||
|
||||
|
||||
def _resolve_as_rename(pending, new_vault_path, new_rel):
|
||||
"""A delete+create pair shared the same JD ID within the grace window
|
||||
-- treat it as the rename it actually is: move /data's old-named
|
||||
folder to the new name (and its folder-note filename to match),
|
||||
instead of archiving. Falls back to a normal create-mirror if /data
|
||||
has no old-named counterpart to move."""
|
||||
old_rel = pending["rel"]
|
||||
data_src = os.path.join(DATA, old_rel)
|
||||
data_dest = os.path.join(DATA, new_rel)
|
||||
if not os.path.isdir(data_src):
|
||||
handle_vault_created(new_vault_path)
|
||||
return
|
||||
if os.path.exists(data_dest):
|
||||
log(f"SKIP rename-recovery mirror (dest exists): {old_rel} -> {new_rel}")
|
||||
return
|
||||
os.makedirs(os.path.dirname(data_dest), exist_ok=True)
|
||||
os.rename(data_src, data_dest)
|
||||
old_base = pending["base"]
|
||||
new_base = os.path.basename(new_vault_path)
|
||||
old_note = os.path.join(data_dest, old_base + ".md")
|
||||
new_note = os.path.join(data_dest, new_base + ".md")
|
||||
if old_base != new_base and os.path.isfile(old_note) and not os.path.exists(new_note):
|
||||
os.rename(old_note, new_note)
|
||||
log(f"RECOVERED rename (delete+create pair, same ID {vl.id_code(new_base)}): {old_rel} -> {new_rel}")
|
||||
ntfy_post(
|
||||
"Vault sync: renamed",
|
||||
f"/data: {old_rel} -> {new_rel}",
|
||||
tags="package",
|
||||
)
|
||||
|
||||
|
||||
def _archive_deleted_id(path, rel, base):
|
||||
data_path = os.path.join(DATA, rel)
|
||||
if not os.path.isdir(data_path):
|
||||
return # nothing to archive -- claimed by a rename, or already gone
|
||||
archive_dir, created_note = _find_nearest_archive_dir(rel)
|
||||
dest = os.path.join(archive_dir, base)
|
||||
if os.path.exists(dest):
|
||||
dest = os.path.join(archive_dir, f"{base} (archived {int(time.time())})")
|
||||
shutil.move(data_path, dest)
|
||||
log(f"ARCHIVED /data: {rel} -> {os.path.relpath(dest, DATA)}")
|
||||
note = _nearest_archive_note(archive_dir)
|
||||
ts = datetime.datetime.now().isoformat(timespec="seconds")
|
||||
entry = (f"\n- **Archived {ts}:** `{base}` (deleted from /vault) — "
|
||||
f"data preserved at `/data/{os.path.relpath(dest, DATA)}`\n")
|
||||
commit_paths = []
|
||||
if note:
|
||||
with open(note, "a", encoding="utf-8") as f:
|
||||
f.write(entry)
|
||||
log(f"NOTED archive in: {os.path.relpath(note, VAULT)}")
|
||||
commit_paths.append(note)
|
||||
if created_note:
|
||||
commit_paths.append(created_note)
|
||||
_git_commit(f"vault_sync_daemon: archived {base}", commit_paths)
|
||||
ntfy_post(
|
||||
"Vault sync: archived",
|
||||
f"Deleted from /vault: {base}\nData preserved at /data/{os.path.relpath(dest, DATA)}",
|
||||
tags="package,wastebasket",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# REVERSE: /data -> /vault
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _id_title(idname):
|
||||
"""'R.32.20 Recreational drugs' -> 'Recreational drugs' (ID stripped)."""
|
||||
m = ID_RE.match(idname)
|
||||
return idname[m.end():].strip() if m else idname
|
||||
|
||||
|
||||
def handle_data_created(path):
|
||||
if not os.path.isdir(path):
|
||||
return
|
||||
if _in_skip(path, DATA):
|
||||
return
|
||||
if not _debounced("data_created", path):
|
||||
return # duplicate inotify event for the same mkdir, ignore
|
||||
rel = os.path.relpath(path, DATA)
|
||||
if _under_data_exempt(rel):
|
||||
return
|
||||
if _under_archive(rel):
|
||||
return # never auto-stub inside archive folders
|
||||
base = os.path.basename(path)
|
||||
if not ID_RE.match(base):
|
||||
return # only ID-shaped folders trigger reverse-sync
|
||||
vault_path = os.path.join(VAULT, rel)
|
||||
if os.path.isdir(vault_path):
|
||||
return # already has a /vault counterpart
|
||||
catcode_root = os.path.dirname(vault_path)
|
||||
if not os.path.isdir(catcode_root):
|
||||
log(f"SKIP stub (parent category folder missing in /vault): {rel}")
|
||||
return
|
||||
note = os.path.join(vault_path, base + ".md")
|
||||
os.makedirs(vault_path, exist_ok=True)
|
||||
alias = _id_title(base)
|
||||
with open(note, "w", encoding="utf-8") as f:
|
||||
f.write(f"---\naliases:\n - {alias}\n---\n")
|
||||
log(f"STUBBED /vault note for new /data folder: {rel}")
|
||||
_git_commit(f"vault_sync_daemon: stubbed {base}", [note])
|
||||
ntfy_post(
|
||||
"Vault sync: new note",
|
||||
f"New /data folder with no vault note: {rel}\nAuto-created stub with alias \"{alias}\".",
|
||||
tags="package,sparkles",
|
||||
)
|
||||
|
||||
|
||||
def handle_data_moved(src_path, dest_path):
|
||||
"""
|
||||
A folder in /data was renamed/moved (e.g. via an SMB client, GUI file
|
||||
manager, or any tool that stages-then-renames rather than mkdir'ing
|
||||
directly in place). DataHandler previously had NO on_moved at all, so
|
||||
this fired zero log line and zero ntfy -- confirmed root cause of
|
||||
Ryan's 2026-09-22 report ("created a folder in /data, nothing happened
|
||||
on ntfy or in /vault": the folder actually arrived via a rename, not a
|
||||
bare create, and there was no handler to catch it).
|
||||
|
||||
Mirror rule: if /vault already has a note at the OLD relative path,
|
||||
rename it to match (same as the forward vault->data rename mirror).
|
||||
If there's no /vault counterpart yet at all, treat this exactly like
|
||||
a fresh data_created at the NEW path (covers the same "staged then
|
||||
renamed into place" arrival pattern as a plain create).
|
||||
"""
|
||||
if _in_skip(src_path, DATA) or _in_skip(dest_path, DATA):
|
||||
return
|
||||
rel_src = os.path.relpath(src_path, DATA)
|
||||
rel_dest = os.path.relpath(dest_path, DATA)
|
||||
if _under_data_exempt(rel_src) or _under_data_exempt(rel_dest):
|
||||
return
|
||||
vault_src = os.path.join(VAULT, rel_src)
|
||||
vault_dest = os.path.join(VAULT, rel_dest)
|
||||
if not os.path.isdir(vault_src):
|
||||
# No /vault counterpart to rename -- treat the arrival at the new
|
||||
# name like a normal reverse-sync create.
|
||||
handle_data_created(dest_path)
|
||||
return
|
||||
if os.path.exists(vault_dest):
|
||||
log(f"SKIP data-move mirror (vault dest exists): {rel_src} -> {rel_dest}")
|
||||
return
|
||||
os.makedirs(os.path.dirname(vault_dest), exist_ok=True)
|
||||
os.rename(vault_src, vault_dest)
|
||||
# keep the folder-note's own filename in sync with its containing
|
||||
# folder's new name (matches the "<ID name>/<ID name>.md" convention
|
||||
# used throughout the vault)
|
||||
old_base = os.path.basename(vault_src)
|
||||
new_base = os.path.basename(vault_dest)
|
||||
old_note = os.path.join(vault_dest, old_base + ".md")
|
||||
new_note = os.path.join(vault_dest, new_base + ".md")
|
||||
if old_base != new_base and os.path.isfile(old_note) and not os.path.exists(new_note):
|
||||
os.rename(old_note, new_note)
|
||||
log(f"MIRRORED rename /vault: {rel_src} -> {rel_dest}")
|
||||
commit_paths = [vault_dest] if os.path.isdir(vault_dest) else []
|
||||
_git_commit(f"vault_sync_daemon: renamed {old_base} -> {new_base}", commit_paths)
|
||||
ntfy_post(
|
||||
"Vault sync: renamed",
|
||||
f"/vault: {rel_src} -> {rel_dest}",
|
||||
tags="package",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# watchdog wiring
|
||||
#
|
||||
# CRITICAL: watchdog's internal reader thread must stay fast. On Linux,
|
||||
# watchdog polls the raw inotify fd and is responsible for pairing a
|
||||
# rename's IN_MOVED_FROM + IN_MOVED_TO events into a single `on_moved`
|
||||
# call (it does this by matching cookies within its own read loop). If our
|
||||
# FileSystemEventHandler callbacks run slow, synchronous work directly on
|
||||
# that thread (git commits, network ntfy calls, time.sleep debounce
|
||||
# delays), a burst of inotify events (e.g. renaming a folder with many
|
||||
# children -> hundreds of individual FS events) can back up the kernel's
|
||||
# inotify event queue faster than watchdog's reader drains it. Once
|
||||
# events start dropping, the MOVED_FROM/MOVED_TO cookie pairing breaks and
|
||||
# watchdog falls back to firing a bare `on_deleted` + a later bare
|
||||
# `on_created` instead of one `on_moved` -- exactly the delete+create
|
||||
# corruption bug Ryan reported (confirmed by reproduction: renaming a
|
||||
# ~300-child folder produced a rename that silently vanished from the log
|
||||
# entirely, while the same rename on a small/empty folder worked fine).
|
||||
#
|
||||
# Fix: handlers do ZERO real work. They just stuff the event onto a
|
||||
# queue and return immediately, so the watchdog reader thread is never
|
||||
# blocked and can drain the kernel's rename event pairs promptly. A
|
||||
# single dedicated worker thread pulls off the queue and does the actual
|
||||
# (slow) sync work, serialized, in FS-event order.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_event_queue = queue.Queue()
|
||||
|
||||
|
||||
class VaultHandler(FileSystemEventHandler):
|
||||
def on_created(self, event):
|
||||
if event.is_directory:
|
||||
_event_queue.put(("vault_created", event.src_path))
|
||||
else:
|
||||
_event_queue.put(("vault_file_created", event.src_path))
|
||||
|
||||
def on_moved(self, event):
|
||||
if not event.is_directory:
|
||||
_event_queue.put(("vault_file_moved", event.src_path, event.dest_path))
|
||||
return
|
||||
_event_queue.put(("vault_moved", event.src_path, event.dest_path))
|
||||
|
||||
def on_deleted(self, event):
|
||||
if not event.is_directory:
|
||||
return
|
||||
_event_queue.put(("vault_deleted", event.src_path))
|
||||
|
||||
|
||||
class DataHandler(FileSystemEventHandler):
|
||||
def on_created(self, event):
|
||||
if not event.is_directory:
|
||||
return
|
||||
_event_queue.put(("data_created", event.src_path))
|
||||
|
||||
def on_moved(self, event):
|
||||
if not event.is_directory:
|
||||
return
|
||||
_event_queue.put(("data_moved", event.src_path, event.dest_path))
|
||||
|
||||
|
||||
def _worker_loop():
|
||||
"""Single consumer thread: drains _event_queue and does the actual
|
||||
(slow) sync work, one event at a time, in the order events arrived.
|
||||
Keeping this off the watchdog reader thread is what lets inotify's
|
||||
MOVED_FROM/MOVED_TO cookie pairing succeed under load (see module
|
||||
docstring above the queue wiring)."""
|
||||
while True:
|
||||
item = _event_queue.get()
|
||||
kind = item[0]
|
||||
try:
|
||||
with _lock:
|
||||
if kind == "vault_created":
|
||||
_, path = item
|
||||
time.sleep(0.5) # let Obsidian finish writing the folder-note
|
||||
handle_vault_created(path)
|
||||
elif kind == "vault_file_created":
|
||||
_, path = item
|
||||
time.sleep(0.5) # let the writer (Obsidian/editor) finish writing content
|
||||
handle_vault_binary_created(path)
|
||||
handle_journal_file_created(path)
|
||||
elif kind == "vault_file_moved":
|
||||
_, src_path, dest_path = item
|
||||
handle_vault_binary_created(dest_path)
|
||||
elif kind == "vault_moved":
|
||||
_, src_path, dest_path = item
|
||||
handle_vault_moved(src_path, dest_path)
|
||||
elif kind == "vault_deleted":
|
||||
_, path = item
|
||||
handle_vault_deleted(path)
|
||||
elif kind == "data_created":
|
||||
_, path = item
|
||||
time.sleep(0.5) # small settle delay so a batch mkdir/copy isn't caught half-written
|
||||
handle_data_created(path)
|
||||
elif kind == "data_moved":
|
||||
_, src_path, dest_path = item
|
||||
handle_data_moved(src_path, dest_path)
|
||||
except Exception as e:
|
||||
log(f"ERROR handling {item}: {e}")
|
||||
ntfy_post(
|
||||
"Vault sync ERROR",
|
||||
f"Handling {kind} failed:\n{item}\n{e}",
|
||||
tags="rotating_light",
|
||||
priority="urgent",
|
||||
)
|
||||
finally:
|
||||
_event_queue.task_done()
|
||||
|
||||
|
||||
RECONCILE_INTERVAL_SECONDS = 180
|
||||
|
||||
|
||||
def reconcile():
|
||||
"""
|
||||
Full-tree self-healing sweep, independent of inotify.
|
||||
|
||||
inotify is best-effort: watch descriptors can be dropped, events can be
|
||||
coalesced or lost under load, and — per Ryan 2026-09-24 — a plain
|
||||
`mkdir` under R.12.22 produced zero log line and zero ntfy with no
|
||||
reproducible cause found (kernel-level inotifywait on the same exact
|
||||
directory caught the same mkdir instantly, so it wasn't a mount/fs
|
||||
issue — watchdog's own bookkeeping simply missed it). Rather than chase
|
||||
every possible way an event can go missing, the daemon must be
|
||||
resilient no matter what: periodically diff the whole tree in both
|
||||
directions and self-heal anything inotify failed to deliver.
|
||||
|
||||
This walks every directory in /data and /vault and calls the exact
|
||||
same idempotent handlers used for live events (handle_data_created /
|
||||
handle_vault_created). Both are "check target state first, act only if
|
||||
something's actually missing" by construction (see module docstring),
|
||||
so calling them on every already-mirrored directory is a cheap no-op —
|
||||
only genuinely missed folders get created, stubbed, and ntfy'd here.
|
||||
|
||||
Deliberately does NOT attempt rename detection: a rename that happened
|
||||
while the daemon was blind looks identical to a delete+create pair,
|
||||
which is genuinely ambiguous to reconstruct after the fact. Live
|
||||
on_moved handling remains the only path for renames; reconcile's job
|
||||
is just to guarantee creates are never permanently lost.
|
||||
"""
|
||||
t0 = time.time()
|
||||
data_count = 0
|
||||
for dirpath, dirnames, _ in os.walk(DATA):
|
||||
dirnames.sort()
|
||||
if dirpath == DATA:
|
||||
continue
|
||||
if _in_skip(dirpath, DATA):
|
||||
dirnames[:] = []
|
||||
continue
|
||||
with _lock:
|
||||
handle_data_created(dirpath)
|
||||
data_count += 1
|
||||
vault_count = 0
|
||||
vault_file_count = 0
|
||||
for dirpath, dirnames, filenames in os.walk(VAULT):
|
||||
dirnames.sort()
|
||||
if dirpath == VAULT:
|
||||
continue
|
||||
if _in_skip(dirpath, VAULT):
|
||||
dirnames[:] = []
|
||||
continue
|
||||
with _lock:
|
||||
handle_vault_created(dirpath)
|
||||
vault_count += 1
|
||||
for fname in filenames:
|
||||
if fname.endswith(".md"):
|
||||
continue
|
||||
with _lock:
|
||||
handle_vault_binary_created(os.path.join(dirpath, fname))
|
||||
vault_file_count += 1
|
||||
log(f"reconcile: swept {data_count} /data dirs, {vault_count} /vault dirs, "
|
||||
f"{vault_file_count} /vault non-md files in {time.time() - t0:.1f}s")
|
||||
|
||||
|
||||
def _reconcile_loop():
|
||||
while True:
|
||||
time.sleep(RECONCILE_INTERVAL_SECONDS)
|
||||
try:
|
||||
reconcile()
|
||||
except Exception as e:
|
||||
log(f"ERROR in reconcile sweep: {e}")
|
||||
ntfy_post(
|
||||
"Vault sync ERROR",
|
||||
f"Periodic reconcile sweep failed:\n{e}",
|
||||
tags="rotating_light",
|
||||
priority="urgent",
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
log("vault_sync_daemon starting")
|
||||
worker = threading.Thread(target=_worker_loop, daemon=True)
|
||||
worker.start()
|
||||
observer = Observer()
|
||||
observer.schedule(VaultHandler(), VAULT, recursive=True)
|
||||
observer.schedule(DataHandler(), DATA, recursive=True)
|
||||
observer.start()
|
||||
log(f"watching {VAULT} and {DATA}")
|
||||
ntfy_post("Vault sync daemon started", f"Watching {VAULT} and {DATA}", tags="rocket", priority="low")
|
||||
# Run one reconcile sweep at startup (catches anything that happened
|
||||
# while the daemon was down, e.g. a reboot) then keep sweeping on a
|
||||
# timer as a standing backstop under live inotify events.
|
||||
try:
|
||||
reconcile()
|
||||
except Exception as e:
|
||||
log(f"ERROR in startup reconcile sweep: {e}")
|
||||
threading.Thread(target=_reconcile_loop, daemon=True).start()
|
||||
try:
|
||||
while True:
|
||||
time.sleep(5)
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
observer.stop()
|
||||
observer.join()
|
||||
log("vault_sync_daemon stopped")
|
||||
# NOTE: no ntfy_post here — a hard kill/OOM/crash skips this `finally`
|
||||
# entirely, so relying on it for "stopped" alerts is unreliable. The
|
||||
# systemd unit's ExecStopPost= hook fires unconditionally on every
|
||||
# exit (clean or not) and is the actual source of stop notifications.
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user