diff --git a/.gitignore b/.gitignore index 61cae5b..576521f 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,10 @@ public/reading_umap.json # (`export.py --explorer`, 15/0.1 plus the kind and section layers). Tracking it would dirty # the tree exactly as above. public/reading_explorer.json +# the /reading/models comparison: a manifest plus one coordinate file per embedding model, +# fetched on demand. Same generator, same reason they must stay untracked. +public/reading_models.json +public/reading_models.*.json .build_state .build.lock .notebooks.lock diff --git a/deploy/auto-build.sh b/deploy/auto-build.sh index 6d6072b..5b772d6 100755 --- a/deploy/auto-build.sh +++ b/deploy/auto-build.sh @@ -30,12 +30,12 @@ sig=$({ git diff find "$VAULT" -name .obsidian -prune -o -type f -printf '%T@ %p\n' | sort find "$REPO/notebooks-export" -type f -printf '%T@ %p\n' 2>/dev/null | sort - # same deal for the reading map: gitignored, written by library-rag-export.service on its - # own timer, so git status/diff above cannot see it change. Both payloads — the banner's and - # the /reading explorer's — or a corpus change would rebuild one page and leave the other - # serving yesterday's cloud. - find "$REPO/public/reading_umap.json" "$REPO/public/reading_explorer.json" \ - -printf '%T@ %p\n' 2>/dev/null | sort + # same deal for every reading map payload: gitignored, written by library-rag-export.service + # on its own timer, so git status/diff above cannot see them change. One glob rather than a + # list — with `-o` the `-printf` binds only to the LAST clause, so an OR-chain silently + # prints the earlier matches in a different format and the signature stops being stable. + # It also means a future payload is watched without editing this line. + find "$REPO/public" -maxdepth 1 -name 'reading_*.json' -printf '%T@ %p\n' 2>/dev/null | sort } | sha256sum | cut -d' ' -f1) if [ "$FORCE" -eq 0 ] && [ -f "$STATE" ] && [ "$(cat "$STATE")" = "$sig" ]; then diff --git a/src/components/ModelCompare.astro b/src/components/ModelCompare.astro new file mode 100644 index 0000000..1aff083 --- /dev/null +++ b/src/components/ModelCompare.astro @@ -0,0 +1,216 @@ +--- +/** + * The model comparison — the same 9,194 passages, arranged by seven different embedding models. + * + * import ModelCompare from '../components/ModelCompare.astro'; + * + * + * Pieces: + * src/components/ModelCompare.astro ← this file + * src/lib/reading-map.js ← the renderer, SHARED with the banner and explorer + * src/styles/reading-map.css ← canvas, every --map-* token + * src/styles/reading-explorer.css ← legend and control chrome + * public/reading_models.json ← manifest; generated, NOT in git + * public/reading_models..json ← one per model; fetched on demand + * + * ⚠️ **The picture is an illustration; the CLAIM is the neighbour-agreement number.** Every + * projection of 1024 dimensions into 3 is lossy in a way that shapes what you see — measured + * directly, PCA and UMAP disagreed by 10× about whether whole books move together, and both were + * reporting their own lens. So the defensible statement is the projection-free one carried in the + * payload (`models[].agreement`), and this page must state it rather than let the animation imply + * something stronger. See library-rag/lab/RESULTS.md. + * + * ⚠️ **Every model's coordinates are Procrustes-aligned to nomic.** Without that the toggle would + * be seven unrelated arrangements and the motion between them meaningless. + * + * ⚠️ Payloads are gitignored and generated on cyrion, exactly like the other two maps. A fresh + * checkout has none and the figure stays hidden. Regenerate with + * `uv run lab/project.py --method umap -o /public/reading_models.json` from library-rag/. + */ +import '../styles/reading-map.css'; +import '../styles/reading-explorer.css'; + +interface Props { + src?: string; + caption?: string | null; +} + +const { + src = '/reading_models.json', + caption = 'The same 9,194 passages, placed by seven different embedding models. Switch between them to see what they disagree about.', +} = Astro.props; +--- + +
+
+ {caption &&
{caption}
} + + + + + +
    +
    + + + + diff --git a/src/lib/reading-map.js b/src/lib/reading-map.js index cf2e1c0..f75e15c 100644 --- a/src/lib/reading-map.js +++ b/src/lib/reading-map.js @@ -149,6 +149,12 @@ export function build(container, data, { explorer = false } = {}) { const xyz = Float32Array.from(data.nodes.xyz); const nodeCount = xyz.length / 3; + // ⚠️ `trace` is optional. The banner and explorer always carry one; the model-comparison + // payload legitimately does not — its subject is where passages MOVE between models, not the + // order they were highlighted in. Defaulting here beats making that caller fabricate an empty + // one, and beats seven `data.trace &&` guards scattered through the draw loop. + if (!Array.isArray(data.trace)) data.trace = []; + // --- the nebula: every chunk, one draw call --------------------------------------------- const nebulaGeo = new THREE.BufferGeometry(); nebulaGeo.setAttribute('position', new THREE.BufferAttribute(xyz, 3)); @@ -505,6 +511,56 @@ export function build(container, data, { explorer = false } = {}) { return isolated; } + // --- model switching (comparison page) ----------------------------------------------------- + /** + * Swap in another model's coordinates for the same points. + * + * ⚠️ **Only legitimate because every model's payload is Procrustes-aligned to the same + * reference** (`lab/project.py`). Without that the clouds share no frame, and what looks like a + * model disagreeing is mostly an arbitrary rotation. Never point this at unaligned coordinates. + * + * `duration` of 0 cuts; anything else eases between the two layouts. Which reads better is an + * open question — p90 displacement is 0.22–0.41 of cloud diameter, so a good fraction of points + * travel a long way and a morph risks looking like a shuffle rather than a drift. + * + * The trace and signal layers follow the nebula, because they index the same rows. + */ + let morph = null; + function setCoordinates(next, duration = 900) { + if (!next || next.length !== xyz.length) return false; + const from = Float32Array.from(xyz); + const to = Float32Array.from(next); + const t0 = performance.now(); + + const apply = (t) => { + // Eased in place: `xyz` is the single source the nebula, signal and trace all read from, + // so writing here moves every layer together and nothing can drift out of registration. + const e = t >= 1 ? 1 : 1 - Math.pow(1 - t, 3); + for (let i = 0; i < xyz.length; i++) xyz[i] = from[i] + (to[i] - from[i]) * e; + nebulaGeo.attributes.position.needsUpdate = true; + signalNodes.forEach((n, i) => { + signalPos[i * 3] = xyz[n * 3]; + signalPos[i * 3 + 1] = xyz[n * 3 + 1]; + signalPos[i * 3 + 2] = xyz[n * 3 + 2]; + }); + signalGeo.attributes.position.needsUpdate = true; + data.trace.forEach((tr, i) => { + tracePos[i * 3] = xyz[tr.node * 3]; + tracePos[i * 3 + 1] = xyz[tr.node * 3 + 1]; + tracePos[i * 3 + 2] = xyz[tr.node * 3 + 2]; + }); + traceGeo.attributes.position.needsUpdate = true; + }; + + if (!duration || reduced) { morph = null; apply(1); return true; } + morph = (now) => { + const t = (now - t0) / duration; + apply(t); + if (t >= 1) morph = null; + }; + return true; + } + // --- resize ------------------------------------------------------------------------------- // ⚠️ Mark ready BEFORE measuring. The fail-closed CSS hides the container until `data-ready` // is set, so a container measured first reports 0×0, the early return leaves the canvas at @@ -541,6 +597,9 @@ export function build(container, data, { explorer = false } = {}) { // homepage, and a continuously rotating scene is exactly what a reader who asked for // reduced motion is asking not to get. The cloud still renders — it just holds still. if (!paused && !reduced) world.rotation.y += ORBIT_RATE * dt; + // Driven from the render loop rather than its own rAF, so a morph cannot outlive the scene + // or keep running while the page is scrolled away (the IntersectionObserver parks this loop). + if (morph) morph(now); elapsed += dt; const cycle = DRAW_SECONDS + HOLD_SECONDS; @@ -549,12 +608,14 @@ export function build(container, data, { explorer = false } = {}) { ? total : Math.max(1, Math.min(total, Math.round((elapsed / DRAW_SECONDS) * total))); - traceGeo.setDrawRange(0, drawn); - const h = (drawn - 1) * 3; - head.geometry.attributes.position.setXYZ(0, tracePos[h], tracePos[h + 1], tracePos[h + 2]); - head.geometry.attributes.position.needsUpdate = true; + if (total) { + traceGeo.setDrawRange(0, drawn); + const h = (drawn - 1) * 3; + head.geometry.attributes.position.setXYZ(0, tracePos[h], tracePos[h + 1], tracePos[h + 2]); + head.geometry.attributes.position.needsUpdate = true; + } - if (drawn - 1 !== shownIndex) { + if (total && drawn - 1 !== shownIndex) { shownIndex = drawn - 1; dateEl.textContent = fmtDate(data.trace[shownIndex].t); } @@ -616,6 +677,7 @@ export function build(container, data, { explorer = false } = {}) { /** Explorer only. A no-op returning null when the payload carries no kind layer. */ isolateBook, get isolatedBook() { return isolated; }, + setCoordinates, /** * What a legend needs, COUNTED FROM THE PAYLOAD rather than hardcoded. A kind's share moves diff --git a/src/pages/reading/models.astro b/src/pages/reading/models.astro new file mode 100644 index 0000000..3cc772f --- /dev/null +++ b/src/pages/reading/models.astro @@ -0,0 +1,71 @@ +--- +import Base from '../../layouts/Base.astro'; +import ModelCompare from '../../components/ModelCompare.astro'; +--- + + +

    Seven models, one library

    + +

    + The reading map shows my library arranged by one embedding model — + the one search actually runs on. This page asks a different question: would another + model arrange it the same way? +

    + + + +

    They retrieve the same. They do not agree.

    + +

    + I scored all seven against 48 hand-written retrieval questions, and the boring result came + first: they are nearly all the same. embeddinggemma ties the incumbent exactly. + Two 568-million-parameter models land within two questions of a 137-million one. Four times + the parameters bought nothing measurable. +

    + +

    + So I asked whether they at least organise the library the same way, and they + emphatically do not. Take any passage, find its fifteen nearest neighbours under two + different models, and compare the lists: they share about half. Between the + incumbent and the weakest model, barely a third. +

    + +

    + What they do agree on is that this is a map of books. Under every single model, + roughly 70–88% of a passage's neighbours come from the same book it does — against 6% if + they were drawn at random. Every model finds the same continents. They disagree about the + streets. +

    + +

    How much of this is real

    + +

    + Squashing 1,024 dimensions into 3 loses most of the information, and it loses it differently + depending on how you squash. When I measured whether whole books move together, two standard + methods disagreed by a factor of ten on identical data — each one confidently reporting a + property of its own arithmetic rather than of the models. So the number quoted above is + computed on the raw embeddings, where there is no projection to be an artifact of, and this + animation is an illustration of that number rather than the evidence for it. +

    + +

    + Some of the motion you see when switching is the layout algorithm rather than the models, + for the same reason. The clouds are rigidly aligned to a common frame first, which removes + the arbitrary rotation but not that last ambiguity. Believe the percentage; enjoy the movement. +

    + +

    + Working notes, including the two occasions this measurement caught me publishing an artifact + as a finding, are in library-rag/lab/RESULTS.md. +

    + + +