wesandClaude Opus 5
9da9a6cba2
Notebooks from the marimo server can now appear in the garden. A notebook
opts in with an HTML comment in one of its markdown cells — invisible when
rendered, greppable in the .py source:
<!-- garden:publish
title: Dream of Spotification
order: 30
-->
Default-deny on purpose: garden.c0smere.net is public and the export bakes
each notebook's executed output into the page, not just its code.
export-notebooks.py also carries a NEVER_PUBLISH list (genome_*, coursework)
so a marker pasted into one of those refuses loudly instead of publishing.
Pipeline: export-notebooks.py runs each marked notebook in a one-shot
marimo container (same image/env/GPU as the live server, so it hits the real
databases) into notebooks-export/ + index.json; copy-notebooks.mjs stages
those to public/nb/; Astro reads index.json to build the pages and the
sidebar section.
Isolation choices worth keeping:
- Own timer and own lock, separate from the 5-min garden build — executing a
notebook takes minutes and must never hold up a build tick.
- Cached on notebook content; both index.json and .cache.json go through
write_if_changed, since auto-build.sh hashes mtimes under notebooks-export/
and an unconditional rewrite would force a full rebuild every 30 minutes.
- Per-notebook timeout; a failure keeps the previous export and continues.
- Runs against a throwaway copy of the notebook dir, so notebooks that write
scratch files don't dirty the notebooks repo.
- Notebooks are NOT injected into the garden collection — they aren't vault
notes, and doing so would move noteCount and the sitemap.
build.format:'file' makes the listing a file (notebooks.html) beside a
directory of detail pages. Verified against the running nginx: /notebooks and
/notebooks/<slug> resolve through the generic try_files but /notebooks/ does
not, so nginx.conf gets an explicit location for the trailing-slash form.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
321 lines
12 KiB
Python
Executable File
321 lines
12 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Export the marimo notebooks that opt in to publication into the garden.
|
|
|
|
A notebook opts in by putting an HTML comment block in one of its markdown
|
|
cells (invisible in the rendered page, greppable in the .py source):
|
|
|
|
<!-- garden:publish
|
|
title: Dream of Spotification
|
|
description: A decade of my own Spotify history, worked two ways.
|
|
order: 20
|
|
-->
|
|
|
|
Only `garden:publish` is required; every other key is optional. Recognized
|
|
keys: title, description, order (sort weight, default 100), slug (URL
|
|
override, default derived from the filename), timeout (seconds, default 600).
|
|
|
|
Nothing publishes without the marker. garden.c0smere.net is public and a
|
|
static export bakes the notebook's *executed output* into the page, not just
|
|
its code — so this is deliberately default-deny, with NEVER_PUBLISH below as
|
|
a second latch under the files where an accidental marker would be worst.
|
|
|
|
Each opted-in notebook is executed against the real databases (same image and
|
|
env as the live marimo server) and snapshotted to notebooks-export/<slug>.html
|
|
next to an index.json the Astro build reads. Exports are cached on notebook
|
|
content, so a run that follows no notebook edits does no work at all.
|
|
|
|
Failure is per-notebook: a broken or hung notebook is logged, its previous
|
|
export is kept, and the run continues — it can never take the garden down.
|
|
"""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from fnmatch import fnmatch
|
|
from pathlib import Path
|
|
|
|
REPO = Path(__file__).resolve().parent.parent
|
|
NOTEBOOK_DIR = Path(os.environ.get("MARIMO_NOTEBOOKS", "/home/nox/docker/marimo/notebooks"))
|
|
MARIMO_ENV = Path(os.environ.get("MARIMO_ENV", "/home/nox/docker/marimo/.env"))
|
|
# The live server's HOME, holding .config/marimo/marimo.toml — the export
|
|
# copies that config in so pages come out in the same theme Wes edits in.
|
|
MARIMO_HOME = Path(os.environ.get("MARIMO_HOME", "/home/nox/docker/marimo/config"))
|
|
MARIMO_IMAGE = os.environ.get("MARIMO_IMAGE", "marimo:latest")
|
|
DOCKER_NETWORK = os.environ.get("MARIMO_NETWORK", "services_net")
|
|
OUT_DIR = REPO / "notebooks-export"
|
|
CACHE_FILE = OUT_DIR / ".cache.json"
|
|
|
|
# Bumped when the export command or output shape changes, to invalidate the
|
|
# cache without anyone having to touch the notebooks.
|
|
EXPORTER_VERSION = "2"
|
|
|
|
# Generous by default: these notebooks fit models against real data, and an
|
|
# export only runs when the notebook actually changed, so a slow one costs
|
|
# nothing on a steady-state tick.
|
|
DEFAULT_TIMEOUT = 1800
|
|
|
|
# Second latch under the marker. These never publish even if one of them
|
|
# picks up a marker by copy-paste: genome_* renders Wes's real genotypes
|
|
# (claude_reader is revoked on that DB precisely so this data stays put), and
|
|
# the coursework file is graded UoPeople work. db.py is a helper, not a
|
|
# notebook. To genuinely publish one of these, remove it from this list —
|
|
# the point is that it takes a deliberate edit here, not just a marker.
|
|
NEVER_PUBLISH = ("genome_*.py", "MTH1211*.py", "db.py")
|
|
|
|
MARKER_RE = re.compile(r"<!--\s*garden:publish\b(?P<body>.*?)-->", re.DOTALL | re.IGNORECASE)
|
|
|
|
|
|
def log(msg):
|
|
print(f"export-notebooks: {msg}", flush=True)
|
|
|
|
|
|
def slugify(text):
|
|
s = re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")
|
|
return s or "notebook"
|
|
|
|
|
|
def parse_marker(source):
|
|
"""Return the marker's key/value dict, or None if the notebook opted out."""
|
|
m = MARKER_RE.search(source)
|
|
if not m:
|
|
return None
|
|
meta = {}
|
|
for line in m.group("body").splitlines():
|
|
line = line.strip()
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
key, sep, value = line.partition(":")
|
|
if not sep:
|
|
continue
|
|
meta[key.strip().lower()] = value.strip()
|
|
return meta
|
|
|
|
|
|
def discover():
|
|
"""All notebooks in the directory, partitioned into published and skipped."""
|
|
published, skipped = [], []
|
|
for path in sorted(NOTEBOOK_DIR.glob("*.py")):
|
|
blocked = any(fnmatch(path.name, pat) for pat in NEVER_PUBLISH)
|
|
try:
|
|
source = path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError as exc:
|
|
log(f"WARNING: cannot read {path.name}: {exc}")
|
|
continue
|
|
meta = parse_marker(source)
|
|
if meta is None:
|
|
skipped.append((path.name, "no marker"))
|
|
continue
|
|
if blocked:
|
|
# Loud: a marker on a denylisted file means someone either
|
|
# copy-pasted it or genuinely meant to change policy.
|
|
log(f"REFUSING {path.name}: matches NEVER_PUBLISH despite carrying a publish marker")
|
|
skipped.append((path.name, "denylisted"))
|
|
continue
|
|
slug = meta.get("slug") or slugify(path.stem)
|
|
try:
|
|
order = int(meta.get("order", 100))
|
|
except ValueError:
|
|
order = 100
|
|
try:
|
|
timeout = int(meta.get("timeout", DEFAULT_TIMEOUT))
|
|
except ValueError:
|
|
timeout = DEFAULT_TIMEOUT
|
|
published.append(
|
|
{
|
|
"slug": slug,
|
|
"source": path.name,
|
|
"title": meta.get("title") or path.stem.replace("_", " ").title(),
|
|
"description": meta.get("description", ""),
|
|
"order": order,
|
|
"timeout": timeout,
|
|
"digest": hashlib.sha256(
|
|
source.encode("utf-8") + f"|{EXPORTER_VERSION}|{slug}".encode()
|
|
).hexdigest(),
|
|
}
|
|
)
|
|
return published, skipped
|
|
|
|
|
|
def staged_workdir(tmp):
|
|
"""A writable copy of the notebook dir.
|
|
|
|
Notebooks import sibling modules (db.py) and some write scratch files
|
|
while running, so the export needs a real working directory — but the
|
|
live notebooks dir is a git repo that gets committed nightly, and export
|
|
runs must not leave anything in it.
|
|
"""
|
|
work = Path(tmp) / "notebooks"
|
|
shutil.copytree(
|
|
NOTEBOOK_DIR,
|
|
work,
|
|
ignore=shutil.ignore_patterns(".git", "__pycache__", "__marimo__", ".gitignore"),
|
|
)
|
|
# Throwaway HOME carrying a copy of the live server's marimo.toml, so the
|
|
# export inherits Wes's theme. A copy rather than a mount of the real
|
|
# config dir: marimo writes cache into HOME, and that must not land in
|
|
# the running server's config.
|
|
home = Path(tmp) / "home"
|
|
(home / ".config" / "marimo").mkdir(parents=True)
|
|
toml = MARIMO_HOME / ".config" / "marimo" / "marimo.toml"
|
|
if toml.is_file():
|
|
shutil.copyfile(toml, home / ".config" / "marimo" / "marimo.toml")
|
|
else:
|
|
log(f"WARNING: no marimo.toml at {toml} — exports will use default theme")
|
|
return work, home
|
|
|
|
|
|
def export_one(nb, work, home, out_dir):
|
|
"""Run one notebook to HTML. Returns True on success."""
|
|
container = f"garden-nb-{nb['slug']}-{os.getpid()}"
|
|
tmp_out = f"{nb['slug']}.html.part"
|
|
cmd = [
|
|
"docker", "run", "--rm", "--name", container,
|
|
"--user", "1000:1000",
|
|
"--network", DOCKER_NETWORK,
|
|
"--env-file", str(MARIMO_ENV),
|
|
# same GPU wiring as the live marimo service — without it xgboost
|
|
# silently falls back to CPU and a fit that takes seconds on the
|
|
# P2000 takes many minutes
|
|
"--runtime", "nvidia",
|
|
"-e", "NVIDIA_VISIBLE_DEVICES=all",
|
|
"-e", "NVIDIA_DRIVER_CAPABILITIES=compute,utility",
|
|
"-e", "HOME=/nbhome",
|
|
"-e", "TZ=America/New_York",
|
|
# notebooks that draw without an explicit backend must not try to
|
|
# open a display inside the container
|
|
"-e", "MPLBACKEND=Agg",
|
|
"-v", f"{work}:/work",
|
|
"-v", f"{home}:/nbhome",
|
|
"-v", f"{out_dir}:/out",
|
|
"-w", "/work",
|
|
MARIMO_IMAGE,
|
|
"marimo", "export", "html", nb["source"], "-o", f"/out/{tmp_out}", "-f",
|
|
]
|
|
started = time.monotonic()
|
|
try:
|
|
proc = subprocess.run(
|
|
cmd, capture_output=True, text=True, timeout=nb["timeout"]
|
|
)
|
|
except subprocess.TimeoutExpired:
|
|
log(f"FAILED {nb['source']}: timed out after {nb['timeout']}s")
|
|
subprocess.run(["docker", "rm", "-f", container], capture_output=True)
|
|
(out_dir / tmp_out).unlink(missing_ok=True)
|
|
return False
|
|
if proc.returncode != 0:
|
|
tail = (proc.stderr or proc.stdout or "").strip().splitlines()[-15:]
|
|
log(f"FAILED {nb['source']}: exit {proc.returncode}")
|
|
for line in tail:
|
|
log(f" | {line}")
|
|
(out_dir / tmp_out).unlink(missing_ok=True)
|
|
return False
|
|
part = out_dir / tmp_out
|
|
if not part.exists() or part.stat().st_size == 0:
|
|
log(f"FAILED {nb['source']}: export produced no output")
|
|
part.unlink(missing_ok=True)
|
|
return False
|
|
# only swap in the new export once it is known-good, so a failure always
|
|
# leaves the previously published page intact
|
|
part.replace(out_dir / f"{nb['slug']}.html")
|
|
log(f"exported {nb['source']} -> {nb['slug']}.html ({time.monotonic() - started:.0f}s)")
|
|
return True
|
|
|
|
|
|
def write_if_changed(path, text):
|
|
"""Avoid rewriting identical files — auto-build.sh hashes mtimes."""
|
|
if path.exists() and path.read_text(encoding="utf-8") == text:
|
|
return False
|
|
path.write_text(text, encoding="utf-8")
|
|
return True
|
|
|
|
|
|
def main():
|
|
force = "--force" in sys.argv
|
|
OUT_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
published, skipped = discover()
|
|
if not NOTEBOOK_DIR.is_dir():
|
|
log(f"notebook dir {NOTEBOOK_DIR} missing — nothing to do")
|
|
return 0
|
|
|
|
try:
|
|
cache = json.loads(CACHE_FILE.read_text())
|
|
except (OSError, ValueError):
|
|
cache = {}
|
|
|
|
live_slugs = {nb["slug"] for nb in published}
|
|
# drop exports whose notebook lost its marker, was renamed, or was deleted
|
|
for stale in OUT_DIR.glob("*.html"):
|
|
if stale.stem not in live_slugs:
|
|
log(f"unpublishing {stale.name} (no longer marked)")
|
|
stale.unlink()
|
|
cache.pop(stale.stem, None)
|
|
|
|
entries, failures = [], 0
|
|
todo = [
|
|
nb for nb in published
|
|
if force
|
|
or cache.get(nb["slug"], {}).get("digest") != nb["digest"]
|
|
or not (OUT_DIR / f"{nb['slug']}.html").exists()
|
|
]
|
|
|
|
if todo:
|
|
with tempfile.TemporaryDirectory(prefix="garden-nb-") as tmp:
|
|
work, home = staged_workdir(tmp)
|
|
for nb in todo:
|
|
if export_one(nb, work, home, OUT_DIR):
|
|
cache[nb["slug"]] = {
|
|
"digest": nb["digest"],
|
|
"exported_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
|
}
|
|
else:
|
|
failures += 1
|
|
|
|
for nb in sorted(published, key=lambda n: (n["order"], n["title"].lower())):
|
|
if not (OUT_DIR / f"{nb['slug']}.html").exists():
|
|
# never exported successfully — leave it out of the index rather
|
|
# than publish a link to a 404
|
|
continue
|
|
cached = cache.get(nb["slug"], {})
|
|
entries.append(
|
|
{
|
|
"slug": nb["slug"],
|
|
"title": nb["title"],
|
|
"description": nb["description"],
|
|
"order": nb["order"],
|
|
"source": nb["source"],
|
|
"exportedAt": cached.get("exported_at"),
|
|
# the notebook changed but its re-export failed: the page is
|
|
# real, just behind the notebook
|
|
"stale": cached.get("digest") != nb["digest"],
|
|
}
|
|
)
|
|
|
|
# Both writes go through write_if_changed: auto-build.sh's signature
|
|
# hashes the mtime of every file under notebooks-export/, dotfiles
|
|
# included, so rewriting an identical cache would force a full garden
|
|
# rebuild on every single export tick.
|
|
write_if_changed(CACHE_FILE, json.dumps(cache, indent=2, sort_keys=True) + "\n")
|
|
changed = write_if_changed(
|
|
OUT_DIR / "index.json", json.dumps(entries, indent=2) + "\n"
|
|
)
|
|
|
|
for name, why in skipped:
|
|
log(f"skipped {name} ({why})")
|
|
log(
|
|
f"{len(entries)} published, {len(skipped)} skipped, {failures} failed"
|
|
f"{', index updated' if changed else ''}"
|
|
)
|
|
# a failed notebook must not fail the run — the garden still builds with
|
|
# whatever exported cleanly. Exit 1 only so systemd surfaces it.
|
|
return 1 if failures else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|