#!/usr/bin/env python3 """Export the marimo notebooks that opt in to publication into the garden. A notebook opts in by putting an HTML comment block in one of its markdown cells (invisible in the rendered page, greppable in the .py source): Only `garden:publish` is required; every other key is optional. Recognized keys: title, description, order (sort weight, default 100), slug (URL override, default derived from the filename), timeout (seconds, default 600). Nothing publishes without the marker. garden.c0smere.net is public and a static export bakes the notebook's *executed output* into the page, not just its code — so this is deliberately default-deny, with NEVER_PUBLISH below as a second latch under the files where an accidental marker would be worst. Each opted-in notebook is executed against the real databases (same image and env as the live marimo server) and snapshotted to notebooks-export/.html next to an index.json the Astro build reads. Exports are cached on notebook content, so a run that follows no notebook edits does no work at all. Failure is per-notebook: a broken or hung notebook is logged, its previous export is kept, and the run continues — it can never take the garden down. """ import hashlib import json import os import re import shutil import subprocess import sys import tempfile import time from datetime import datetime, timezone from fnmatch import fnmatch from pathlib import Path REPO = Path(__file__).resolve().parent.parent NOTEBOOK_DIR = Path(os.environ.get("MARIMO_NOTEBOOKS", "/home/nox/docker/marimo/notebooks")) MARIMO_ENV = Path(os.environ.get("MARIMO_ENV", "/home/nox/docker/marimo/.env")) # The live server's HOME, holding .config/marimo/marimo.toml — the export # copies that config in so pages come out in the same theme Wes edits in. MARIMO_HOME = Path(os.environ.get("MARIMO_HOME", "/home/nox/docker/marimo/config")) MARIMO_IMAGE = os.environ.get("MARIMO_IMAGE", "marimo:latest") DOCKER_NETWORK = os.environ.get("MARIMO_NETWORK", "services_net") OUT_DIR = REPO / "notebooks-export" CACHE_FILE = OUT_DIR / ".cache.json" # Bumped when the export command or output shape changes, to invalidate the # cache without anyone having to touch the notebooks. EXPORTER_VERSION = "2" # Generous by default: these notebooks fit models against real data, and an # export only runs when the notebook actually changed, so a slow one costs # nothing on a steady-state tick. DEFAULT_TIMEOUT = 1800 # Second latch under the marker. These never publish even if one of them # picks up a marker by copy-paste: genome_* renders Wes's real genotypes # (claude_reader is revoked on that DB precisely so this data stays put), and # the coursework file is graded UoPeople work. db.py is a helper, not a # notebook. To genuinely publish one of these, remove it from this list — # the point is that it takes a deliberate edit here, not just a marker. NEVER_PUBLISH = ("genome_*.py", "MTH1211*.py", "db.py") MARKER_RE = re.compile(r"", re.DOTALL | re.IGNORECASE) def log(msg): print(f"export-notebooks: {msg}", flush=True) def slugify(text): s = re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-") return s or "notebook" def parse_marker(source): """Return the marker's key/value dict, or None if the notebook opted out.""" m = MARKER_RE.search(source) if not m: return None meta = {} for line in m.group("body").splitlines(): line = line.strip() if not line or line.startswith("#"): continue key, sep, value = line.partition(":") if not sep: continue meta[key.strip().lower()] = value.strip() return meta def discover(): """All notebooks in the directory, partitioned into published and skipped.""" published, skipped = [], [] for path in sorted(NOTEBOOK_DIR.glob("*.py")): blocked = any(fnmatch(path.name, pat) for pat in NEVER_PUBLISH) try: source = path.read_text(encoding="utf-8", errors="replace") except OSError as exc: log(f"WARNING: cannot read {path.name}: {exc}") continue meta = parse_marker(source) if meta is None: skipped.append((path.name, "no marker")) continue if blocked: # Loud: a marker on a denylisted file means someone either # copy-pasted it or genuinely meant to change policy. log(f"REFUSING {path.name}: matches NEVER_PUBLISH despite carrying a publish marker") skipped.append((path.name, "denylisted")) continue slug = meta.get("slug") or slugify(path.stem) try: order = int(meta.get("order", 100)) except ValueError: order = 100 try: timeout = int(meta.get("timeout", DEFAULT_TIMEOUT)) except ValueError: timeout = DEFAULT_TIMEOUT published.append( { "slug": slug, "source": path.name, "title": meta.get("title") or path.stem.replace("_", " ").title(), "description": meta.get("description", ""), "order": order, "timeout": timeout, "digest": hashlib.sha256( source.encode("utf-8") + f"|{EXPORTER_VERSION}|{slug}".encode() ).hexdigest(), } ) return published, skipped def staged_workdir(tmp): """A writable copy of the notebook dir. Notebooks import sibling modules (db.py) and some write scratch files while running, so the export needs a real working directory — but the live notebooks dir is a git repo that gets committed nightly, and export runs must not leave anything in it. """ work = Path(tmp) / "notebooks" shutil.copytree( NOTEBOOK_DIR, work, ignore=shutil.ignore_patterns(".git", "__pycache__", "__marimo__", ".gitignore"), ) # Throwaway HOME carrying a copy of the live server's marimo.toml, so the # export inherits Wes's theme. A copy rather than a mount of the real # config dir: marimo writes cache into HOME, and that must not land in # the running server's config. home = Path(tmp) / "home" (home / ".config" / "marimo").mkdir(parents=True) toml = MARIMO_HOME / ".config" / "marimo" / "marimo.toml" if toml.is_file(): shutil.copyfile(toml, home / ".config" / "marimo" / "marimo.toml") else: log(f"WARNING: no marimo.toml at {toml} — exports will use default theme") return work, home def export_one(nb, work, home, out_dir): """Run one notebook to HTML. Returns True on success.""" container = f"garden-nb-{nb['slug']}-{os.getpid()}" tmp_out = f"{nb['slug']}.html.part" cmd = [ "docker", "run", "--rm", "--name", container, "--user", "1000:1000", "--network", DOCKER_NETWORK, "--env-file", str(MARIMO_ENV), # same GPU wiring as the live marimo service — without it xgboost # silently falls back to CPU and a fit that takes seconds on the # P2000 takes many minutes "--runtime", "nvidia", "-e", "NVIDIA_VISIBLE_DEVICES=all", "-e", "NVIDIA_DRIVER_CAPABILITIES=compute,utility", "-e", "HOME=/nbhome", "-e", "TZ=America/New_York", # notebooks that draw without an explicit backend must not try to # open a display inside the container "-e", "MPLBACKEND=Agg", "-v", f"{work}:/work", "-v", f"{home}:/nbhome", "-v", f"{out_dir}:/out", "-w", "/work", MARIMO_IMAGE, "marimo", "export", "html", nb["source"], "-o", f"/out/{tmp_out}", "-f", ] started = time.monotonic() try: proc = subprocess.run( cmd, capture_output=True, text=True, timeout=nb["timeout"] ) except subprocess.TimeoutExpired: log(f"FAILED {nb['source']}: timed out after {nb['timeout']}s") subprocess.run(["docker", "rm", "-f", container], capture_output=True) (out_dir / tmp_out).unlink(missing_ok=True) return False if proc.returncode != 0: tail = (proc.stderr or proc.stdout or "").strip().splitlines()[-15:] log(f"FAILED {nb['source']}: exit {proc.returncode}") for line in tail: log(f" | {line}") (out_dir / tmp_out).unlink(missing_ok=True) return False part = out_dir / tmp_out if not part.exists() or part.stat().st_size == 0: log(f"FAILED {nb['source']}: export produced no output") part.unlink(missing_ok=True) return False # only swap in the new export once it is known-good, so a failure always # leaves the previously published page intact part.replace(out_dir / f"{nb['slug']}.html") log(f"exported {nb['source']} -> {nb['slug']}.html ({time.monotonic() - started:.0f}s)") return True def write_if_changed(path, text): """Avoid rewriting identical files — auto-build.sh hashes mtimes.""" if path.exists() and path.read_text(encoding="utf-8") == text: return False path.write_text(text, encoding="utf-8") return True def main(): force = "--force" in sys.argv OUT_DIR.mkdir(parents=True, exist_ok=True) published, skipped = discover() if not NOTEBOOK_DIR.is_dir(): log(f"notebook dir {NOTEBOOK_DIR} missing — nothing to do") return 0 try: cache = json.loads(CACHE_FILE.read_text()) except (OSError, ValueError): cache = {} live_slugs = {nb["slug"] for nb in published} # drop exports whose notebook lost its marker, was renamed, or was deleted for stale in OUT_DIR.glob("*.html"): if stale.stem not in live_slugs: log(f"unpublishing {stale.name} (no longer marked)") stale.unlink() cache.pop(stale.stem, None) entries, failures = [], 0 todo = [ nb for nb in published if force or cache.get(nb["slug"], {}).get("digest") != nb["digest"] or not (OUT_DIR / f"{nb['slug']}.html").exists() ] if todo: with tempfile.TemporaryDirectory(prefix="garden-nb-") as tmp: work, home = staged_workdir(tmp) for nb in todo: if export_one(nb, work, home, OUT_DIR): cache[nb["slug"]] = { "digest": nb["digest"], "exported_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), } else: failures += 1 for nb in sorted(published, key=lambda n: (n["order"], n["title"].lower())): if not (OUT_DIR / f"{nb['slug']}.html").exists(): # never exported successfully — leave it out of the index rather # than publish a link to a 404 continue cached = cache.get(nb["slug"], {}) entries.append( { "slug": nb["slug"], "title": nb["title"], "description": nb["description"], "order": nb["order"], "source": nb["source"], "exportedAt": cached.get("exported_at"), # the notebook changed but its re-export failed: the page is # real, just behind the notebook "stale": cached.get("digest") != nb["digest"], } ) # Both writes go through write_if_changed: auto-build.sh's signature # hashes the mtime of every file under notebooks-export/, dotfiles # included, so rewriting an identical cache would force a full garden # rebuild on every single export tick. write_if_changed(CACHE_FILE, json.dumps(cache, indent=2, sort_keys=True) + "\n") changed = write_if_changed( OUT_DIR / "index.json", json.dumps(entries, indent=2) + "\n" ) for name, why in skipped: log(f"skipped {name} ({why})") log( f"{len(entries)} published, {len(skipped)} skipped, {failures} failed" f"{', index updated' if changed else ''}" ) # a failed notebook must not fail the run — the garden still builds with # whatever exported cleanly. Exit 1 only so systemd surfaces it. return 1 if failures else 0 if __name__ == "__main__": sys.exit(main())