// Stage · 3D — residual-stream viewer. // // Lives inside the expanded residual-stream panel (RainBackdrop in app.jsx). // Renders the residual stream as a 3D volume: // X axis — token position (left → right) // Z axis — layer depth (front → back, 0 … nLayers-1) // Y axis — activation magnitude (ribbon height) // // Design ruleset (from autoplan design review, locked): // - MeshBasicMaterial only. No lights, no shadows. // - No textures. Wireframes + solid-color faces. // - FogExp2 tuned to var(--bg) so far cells dissolve into the page bg. // - All THREE.Color instances read from oklch CSS vars — no hex literals. // - Custom minimal orbit (no drei / OrbitControls dep). // - Scanline overlay from index.html::after stays visible on top. const N_LAYERS = 32; const N_TOKENS = 8; // Load THREE on demand. Resolves once window.THREE is available (the module // script in index.html fires `threeLoaded`). function awaitThree() { if (typeof window === 'undefined') return Promise.reject(new Error('no window')); if (window.THREE) return Promise.resolve(window.THREE); return new Promise((resolve) => { window.addEventListener('threeLoaded', () => resolve(window.THREE), { once: true }); }); } // Read an oklch CSS variable and convert it into a THREE.Color. Evaluated // once at scene setup — document.documentElement's computed styles reflect // the :root custom properties from index.html. Offscreen span lets the // browser resolve oklch() → sRGB for us (no manual color conversion math). function cssVarToThreeColor(THREE, varName, fallback = '#ffffff') { const value = getComputedStyle(document.documentElement).getPropertyValue(varName).trim() || fallback; const probe = document.createElement('span'); probe.style.color = value; probe.style.display = 'none'; document.body.appendChild(probe); const rgb = getComputedStyle(probe).color; document.body.removeChild(probe); const m = rgb.match(/rgba?\(([^)]+)\)/); if (!m) return new THREE.Color(1, 1, 1); const [r, g, b] = m[1].split(',').map(s => parseFloat(s.trim()) / 255); return new THREE.Color(r, g, b); } // Deterministic per-(token, layer) magnitude. Shared with 2D Stage 03/04 // through window.residualRNG (see utils.jsx) so the values line up across // views. Local alias kept for readability in the scene geometry code. function residualMagnitude(tokenIdx, layer, seed = 1) { return window.residualRNG(tokenIdx, layer, 0, seed); } function ResidualScene({ active, tokens, stageRef, progressRef }) { const hostRef = React.useRef(null); const [status, setStatus] = React.useState('loading'); // 'loading' | 'ready' | 'error' | 'no-webgl' const [hint, setHint] = React.useState(null); // { token, layer } React.useEffect(() => { if (!active) return; let cancelled = false; let renderer, scene, camera, raf, cleanupFns = []; (async () => { const probe = document.createElement('canvas'); const gl = probe.getContext('webgl2') || probe.getContext('webgl'); if (!gl) { if (!cancelled) setStatus('no-webgl'); return; } let THREE; try { THREE = await awaitThree(); } catch { if (!cancelled) setStatus('error'); return; } if (cancelled) return; const host = hostRef.current; if (!host) return; let width = host.clientWidth; let height = host.clientHeight; if (!width || !height) { if (!cancelled) setStatus('error'); return; } // ── Scene + camera ───────────────────────────────────────────── scene = new THREE.Scene(); const bg = cssVarToThreeColor(THREE, '--bg', '#0a0d0a'); scene.fog = new THREE.FogExp2(bg.getHex(), 0.004); // Orbit state (spherical coords around origin). Camera position is // computed every frame from these. Auto-rotate drifts azimuth when idle. const orbit = { azimuth: 0.3, // radians — left/right around Y polar: 1.1, // radians — up/down from +Y (clamped 0.2…1.5) radius: 380, // distance from origin (clamped 150…700) targetY: 2.5, // lookAt y-offset (keeps grid centered) autoRotate: true, // drifts azimuth when no user input lastInteractAt: 0, // ms epoch; auto-rotate resumes 2500ms after last drag }; camera = new THREE.PerspectiveCamera(56, width / height, 1, 2000); const applyCamera = () => { const x = Math.sin(orbit.azimuth) * Math.sin(orbit.polar) * orbit.radius; const z = Math.cos(orbit.azimuth) * Math.sin(orbit.polar) * orbit.radius; const y = Math.cos(orbit.polar) * orbit.radius; camera.position.set(x, y, z); camera.lookAt(0, orbit.targetY, 0); }; applyCamera(); renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true, powerPreference: 'high-performance', }); renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2)); renderer.setSize(width, height, false); renderer.setClearColor(bg.getHex(), 1); host.appendChild(renderer.domElement); Object.assign(renderer.domElement.style, { display: 'block', width: '100%', height: '100%', cursor: 'grab', touchAction: 'none', userSelect: 'none', }); // ── Palette ─────────────────────────────────────────────────── const inkColor = cssVarToThreeColor(THREE, '--ink', '#a0ffb0'); const amberColor = cssVarToThreeColor(THREE, '--amber', '#ffb060'); const magentaColor = cssVarToThreeColor(THREE, '--magenta', '#e060b0'); // ── Lattice geometry ────────────────────────────────────────── const spacingX = 26; const spacingZ = 11; const totalX = spacingX * (N_TOKENS - 1); const totalZ = spacingZ * (N_LAYERS - 1); const cellGeo = new THREE.BoxGeometry(12, 3, 9); const cellMat = new THREE.MeshBasicMaterial({ color: inkColor, transparent: true, opacity: 0.24, }); const cells = new THREE.InstancedMesh(cellGeo, cellMat, N_LAYERS * N_TOKENS); cells.instanceMatrix.setUsage(THREE.DynamicDrawUsage); const edgePositions = []; const dummy = new THREE.Object3D(); const activeTokenIdx = Math.max(0, (tokens?.length ?? 1) - 1); // Per-cell metadata so hover/highlight can reach each instance cheaply. const cellMeta = []; // index → { t, l, x, y, z, yScale } for (let l = 0; l < N_LAYERS; l++) { for (let t = 0; t < N_TOKENS; t++) { const idx = l * N_TOKENS + t; const x = t * spacingX - totalX / 2; const z = l * spacingZ - totalZ / 2; const mag = residualMagnitude(t, l); const yScale = 0.3 + mag * 2.2; const yPos = yScale * 2; dummy.position.set(x, yPos, z); dummy.scale.set(1, yScale, 1); dummy.rotation.set(0, 0, 0); dummy.updateMatrix(); cells.setMatrixAt(idx, dummy.matrix); const col = t === activeTokenIdx ? magentaColor : inkColor; cells.setColorAt && cells.setColorAt(idx, col); cellMeta.push({ t, l, x, y: yPos, z, yScale }); // Merged wireframe edges const hx = 6, hy = yScale * 1.5, hz = 4.5; const corners = [ [x - hx, yPos - hy, z - hz], [x + hx, yPos - hy, z - hz], [x + hx, yPos - hy, z + hz], [x - hx, yPos - hy, z + hz], [x - hx, yPos + hy, z - hz], [x + hx, yPos + hy, z - hz], [x + hx, yPos + hy, z + hz], [x - hx, yPos + hy, z + hz], ]; const edgeIdx = [ [0,1],[1,2],[2,3],[3,0], [4,5],[5,6],[6,7],[7,4], [0,4],[1,5],[2,6],[3,7], ]; for (const [a, b] of edgeIdx) { edgePositions.push(...corners[a], ...corners[b]); } } } cells.instanceMatrix.needsUpdate = true; if (cells.instanceColor) cells.instanceColor.needsUpdate = true; scene.add(cells); const edgeGeo = new THREE.BufferGeometry(); edgeGeo.setAttribute('position', new THREE.Float32BufferAttribute(edgePositions, 3)); const edgeMat = new THREE.LineBasicMaterial({ color: inkColor, transparent: true, opacity: 0.55, }); scene.add(new THREE.LineSegments(edgeGeo, edgeMat)); // Axis guides const axisMat = new THREE.LineBasicMaterial({ color: inkColor, transparent: true, opacity: 0.14, }); const ax = totalX / 2 + spacingX * 0.6, az = totalZ / 2 + spacingZ * 0.6; const axisGeo = new THREE.BufferGeometry(); axisGeo.setAttribute('position', new THREE.Float32BufferAttribute([ -ax, 0, -az, ax, 0, -az, ax, 0, -az, ax, 0, az, -ax, 0, -az, -ax, 0, az, -ax, 0, az, ax, 0, az, ], 3)); scene.add(new THREE.LineSegments(axisGeo, axisMat)); // Token-column highlight indicator — a subtle vertical line above each // token's column, lit bright when hovered/selected. const highlightMat = new THREE.LineBasicMaterial({ color: amberColor, transparent: true, opacity: 0, }); const highlightGeo = new THREE.BufferGeometry(); highlightGeo.setAttribute('position', new THREE.Float32BufferAttribute([0, -20, 0, 0, 40, 0], 3)); const hoverPillar = new THREE.LineSegments(highlightGeo, highlightMat); scene.add(hoverPillar); // ── Attention arcs ───────────────────────────────────────────── // One aggregated arc per (source_token, layer) → (active_token, layer). // Per-head + per-layer would be ~1700 splines; that overwhelms the // pedagogy ("look at the flow") with noise. Aggregating across heads // per layer keeps the signal ("this layer attended strongly to that // past token") and the count (~256 arcs max) stays affordable. // // Each arc = a 12-segment Catmull-Rom spline rendered as a Line. // Color = amber tinted by aggregate attention weight; opacity likewise. // Arcs drawn only from tokens < activeTokenIdx (causal mask). const arcSegments = 12; const arcPositions = []; const arcColors = []; if (activeTokenIdx > 0) { for (let l = 0; l < N_LAYERS; l++) { const zLayer = l * spacingZ - totalZ / 2; const targetY = residualMagnitude(activeTokenIdx, l) * 4.4 + 0.6; const targetX = activeTokenIdx * spacingX - totalX / 2; for (let srcT = 0; srcT < activeTokenIdx; srcT++) { // Aggregated head weight from this (src, activeTok, layer) triple. // Deterministic, seeded by (srcT, activeTok, layer). let weight = 0; for (let h = 0; h < 8; h++) { weight += residualMagnitude(srcT * 8 + h, l, 3) * residualMagnitude(activeTokenIdx, l, 7); } weight = weight / 8; if (weight < 0.15) continue; // prune the faintest (reduces visual noise) const srcX = srcT * spacingX - totalX / 2; const srcY = residualMagnitude(srcT, l) * 4.4 + 0.6; // Control point lifted above the layer's Z-plane for a readable arch. const midX = (srcX + targetX) / 2; const midY = Math.max(srcY, targetY) + 8 + weight * 6; const midZ = zLayer; // Sample the Catmull-Rom spline at arcSegments+1 points → arcSegments line segments. const p0 = new THREE.Vector3(srcX, srcY, zLayer); const p3 = new THREE.Vector3(targetX, targetY, zLayer); const p1 = new THREE.Vector3(srcX, srcY + 4, zLayer); const p2 = new THREE.Vector3(targetX, targetY + 4, zLayer); const curve = new THREE.CubicBezierCurve3( p0, new THREE.Vector3(midX - 4, midY, midZ), new THREE.Vector3(midX + 4, midY, midZ), p3 ); const pts = curve.getPoints(arcSegments); // Build line-segment pairs from the sampled points. const arcAlpha = Math.min(1, weight * 1.8); for (let i = 0; i < pts.length - 1; i++) { arcPositions.push(pts[i].x, pts[i].y, pts[i].z); arcPositions.push(pts[i + 1].x, pts[i + 1].y, pts[i + 1].z); // Vertex colors = amber scaled by arcAlpha (since LineBasicMaterial // doesn't support per-segment opacity, we fake it by dimming the // color towards bg for low-weight arcs). const a = arcAlpha; for (let j = 0; j < 2; j++) { arcColors.push( amberColor.r * a, amberColor.g * a * 0.75, // slightly less green in the amber amberColor.b * a * 0.4, ); } } } } } let arcMat = null; if (arcPositions.length > 0) { const arcGeo = new THREE.BufferGeometry(); arcGeo.setAttribute('position', new THREE.Float32BufferAttribute(arcPositions, 3)); arcGeo.setAttribute('color', new THREE.Float32BufferAttribute(arcColors, 3)); arcMat = new THREE.LineBasicMaterial({ vertexColors: true, transparent: true, opacity: 0.3, }); scene.add(new THREE.LineSegments(arcGeo, arcMat)); } // ── MLP write indicators ─────────────────────────────────────── // Each cell gets a short vertical line rising from its top, length and // opacity proportional to the cell's MLP write strength for that // (token, layer) pair. Visually: "this cell is where the MLP added // signal to the residual stream at this layer." Merged into a single // LineSegments geometry. const mlpPositions = []; const mlpColors = []; for (let l = 0; l < N_LAYERS; l++) { for (let t = 0; t < N_TOKENS; t++) { const meta = cellMeta[l * N_TOKENS + t]; const strength = residualMagnitude(t, l, 11); // seed=11 → distinct from magnitude/attention const topY = meta.y + meta.yScale * 1.5; const tipY = topY + 2 + strength * 5; mlpPositions.push(meta.x, topY, meta.z); mlpPositions.push(meta.x, tipY, meta.z); // Brighter phosphor at the tip, fading down — gradient via vertex // colors (THREE interpolates across each segment automatically). mlpColors.push(inkColor.r * 0.4, inkColor.g * 0.4, inkColor.b * 0.4); mlpColors.push(inkColor.r, inkColor.g, inkColor.b); } } const mlpGeo = new THREE.BufferGeometry(); mlpGeo.setAttribute('position', new THREE.Float32BufferAttribute(mlpPositions, 3)); mlpGeo.setAttribute('color', new THREE.Float32BufferAttribute(mlpColors, 3)); const mlpMat = new THREE.LineBasicMaterial({ vertexColors: true, transparent: true, opacity: 0.3, }); scene.add(new THREE.LineSegments(mlpGeo, mlpMat)); setStatus('ready'); // ── Input: drag to orbit, scroll to zoom, click-vs-drag gate ─── const canvas = renderer.domElement; let isDragging = false; let dragStart = { x: 0, y: 0 }; let lastMove = { x: 0, y: 0 }; let movedEnough = false; const DRAG_THRESHOLD_PX = 4; const onPointerDown = (e) => { if (e.button !== undefined && e.button !== 0) return; canvas.setPointerCapture?.(e.pointerId); isDragging = true; movedEnough = false; dragStart = { x: e.clientX, y: e.clientY }; lastMove = dragStart; canvas.style.cursor = 'grabbing'; orbit.autoRotate = false; orbit.lastInteractAt = performance.now(); }; const onPointerMove = (e) => { // Track hover column for highlight even when not dragging. const r = canvas.getBoundingClientRect(); const mx = e.clientX - r.left; // Map mouse x to nearest token column via camera projection of each // column center. We don't need raycasting precision — token columns // are spaced far apart in world X. const ndcX = (mx / r.width) * 2 - 1; // Cheap estimate: pick the column whose projected X is closest to ndcX. let bestT = -1, bestDist = 1e9; const tmp = new THREE.Vector3(); for (let t = 0; t < N_TOKENS; t++) { tmp.set(t * spacingX - totalX / 2, orbit.targetY, 0).project(camera); const d = Math.abs(tmp.x - ndcX); if (d < bestDist) { bestDist = d; bestT = t; } } if (bestT !== -1) setHint((h) => (h?.token === bestT ? h : { token: bestT, layer: h?.layer ?? null })); if (!isDragging) return; const dx = e.clientX - lastMove.x; const dy = e.clientY - lastMove.y; lastMove = { x: e.clientX, y: e.clientY }; const totalMoved = Math.hypot(e.clientX - dragStart.x, e.clientY - dragStart.y); if (totalMoved > DRAG_THRESHOLD_PX) movedEnough = true; orbit.azimuth -= dx * 0.005; orbit.polar -= dy * 0.005; orbit.polar = Math.max(0.2, Math.min(1.5, orbit.polar)); orbit.lastInteractAt = performance.now(); }; const onPointerUp = (e) => { isDragging = false; canvas.style.cursor = 'grab'; canvas.releasePointerCapture?.(e.pointerId); }; const onPointerLeave = () => setHint(null); // Suppress the click-to-collapse bubbling up to the parent if this // was an orbit drag, not a tap. Click event fires after mouseup. const onClick = (e) => { if (movedEnough) { e.stopPropagation(); movedEnough = false; } }; const onWheel = (e) => { e.preventDefault(); const zoomFactor = e.deltaY > 0 ? 1.08 : 1 / 1.08; orbit.radius = Math.max(150, Math.min(700, orbit.radius * zoomFactor)); orbit.lastInteractAt = performance.now(); orbit.autoRotate = false; }; canvas.addEventListener('pointerdown', onPointerDown); canvas.addEventListener('pointermove', onPointerMove); canvas.addEventListener('pointerup', onPointerUp); canvas.addEventListener('pointercancel', onPointerUp); canvas.addEventListener('pointerleave', onPointerLeave); canvas.addEventListener('click', onClick, true); // capture phase, beats parent bubble canvas.addEventListener('wheel', onWheel, { passive: false }); cleanupFns.push(() => { canvas.removeEventListener('pointerdown', onPointerDown); canvas.removeEventListener('pointermove', onPointerMove); canvas.removeEventListener('pointerup', onPointerUp); canvas.removeEventListener('pointercancel', onPointerUp); canvas.removeEventListener('pointerleave', onPointerLeave); canvas.removeEventListener('click', onClick, true); canvas.removeEventListener('wheel', onWheel); }); // ── Keyboard: ←/→ (token), ↑/↓ (layer), space (toggle auto-rotate). // Scope: capture phase on window so the App's keydown handler doesn't // also fire and move the pipeline stage while the scene is open. let kbToken = activeTokenIdx; let kbLayer = 0; const onKey = (e) => { const t = e.target; if (t && (t.tagName === 'INPUT' || t.tagName === 'TEXTAREA' || t.isContentEditable)) return; const keys = ['ArrowLeft', 'ArrowRight', 'ArrowUp', 'ArrowDown', ' ']; if (!keys.includes(e.key)) return; e.preventDefault(); e.stopPropagation(); orbit.lastInteractAt = performance.now(); if (e.key === 'ArrowLeft') { kbToken = Math.max(0, kbToken - 1); setHint({ token: kbToken, layer: kbLayer }); } if (e.key === 'ArrowRight') { kbToken = Math.min(N_TOKENS - 1, kbToken + 1); setHint({ token: kbToken, layer: kbLayer }); } if (e.key === 'ArrowUp') { kbLayer = Math.min(N_LAYERS - 1, kbLayer + 1); setHint({ token: kbToken, layer: kbLayer }); } if (e.key === 'ArrowDown') { kbLayer = Math.max(0, kbLayer - 1); setHint({ token: kbToken, layer: kbLayer }); } if (e.key === ' ') orbit.autoRotate = !orbit.autoRotate; }; window.addEventListener('keydown', onKey, true); cleanupFns.push(() => window.removeEventListener('keydown', onKey, true)); // ── Render loop ─────────────────────────────────────────────── // Live sync: arcs pulse during Stage 03 (attention), MLP lines pulse // during Stage 04. Smoothed lerp so stage transitions feel connected // rather than flickering. Pipeline refs accessed through .current so // RAF loop reads live values without re-subscribing. const ARC_BASE = 0.25; const ARC_ACTIVE = 0.95; const MLP_BASE = 0.25; const MLP_ACTIVE = 0.9; let arcOpacityTarget = ARC_BASE; let mlpOpacityTarget = MLP_BASE; const tick = (now) => { const idleFor = now - orbit.lastInteractAt; if (orbit.autoRotate && (orbit.lastInteractAt === 0 || idleFor > 2500)) { orbit.azimuth += 0.0022; } applyCamera(); // Pipeline-driven opacity modulation. if (stageRef && progressRef) { const s = stageRef.current; const p = progressRef.current; // Stage 2 = attention (0-indexed). Arcs brighten in over progress. if (s === 2) { arcOpacityTarget = ARC_BASE + (ARC_ACTIVE - ARC_BASE) * p; } else if (s === 3) { // Stage just completed — stay bright one stage, then fall back. arcOpacityTarget = ARC_ACTIVE * 0.7; } else { arcOpacityTarget = ARC_BASE; } // Stage 3 = MLP. MLP writes pulse in. if (s === 3) { mlpOpacityTarget = MLP_BASE + (MLP_ACTIVE - MLP_BASE) * p; } else if (s === 4) { mlpOpacityTarget = MLP_ACTIVE * 0.7; } else { mlpOpacityTarget = MLP_BASE; } } // Smooth toward target (exponential decay, ~150ms half-life at 60fps). if (arcMat) arcMat.opacity += (arcOpacityTarget - arcMat.opacity) * 0.08; mlpMat.opacity += (mlpOpacityTarget - mlpMat.opacity) * 0.08; // Hover pillar tracks the current hint's token column. const hintToken = (typeof hint === 'object' && hint && hint.token != null) ? hint.token : -1; if (hintToken >= 0) { const x = hintToken * spacingX - totalX / 2; hoverPillar.position.set(x, orbit.targetY, 0); highlightMat.opacity = 0.55; } else { highlightMat.opacity = 0; } renderer.render(scene, camera); raf = requestAnimationFrame(tick); }; raf = requestAnimationFrame(tick); // ── Resize ──────────────────────────────────────────────────── const ro = new ResizeObserver(() => { if (!host) return; width = host.clientWidth; height = host.clientHeight; if (width && height) { renderer.setSize(width, height, false); camera.aspect = width / height; camera.updateProjectionMatrix(); } }); ro.observe(host); cleanupFns.push(() => ro.disconnect()); renderer.domElement.addEventListener('webglcontextlost', (e) => { e.preventDefault(); if (!cancelled) setStatus('error'); }); })(); return () => { cancelled = true; if (raf) cancelAnimationFrame(raf); cleanupFns.forEach(fn => { try { fn(); } catch {} }); if (renderer) { renderer.dispose(); if (renderer.domElement && renderer.domElement.parentNode) { renderer.domElement.parentNode.removeChild(renderer.domElement); } } if (scene) { scene.traverse((obj) => { if (obj.geometry) obj.geometry.dispose(); if (obj.material) { const mats = Array.isArray(obj.material) ? obj.material : [obj.material]; mats.forEach(m => m.dispose()); } }); } }; // We intentionally depend only on `active` — re-running on token change // would tear down the scene. Future M5 will refresh instance colors // from a stage ref without restart. // eslint-disable-next-line react-hooks/exhaustive-deps }, [active]); // Token/layer read-out when hovering or key-selecting. Lives above the // canvas as a subtle HUD, not blocking pointer events. const showHint = hint && hint.token != null; const hintText = showHint ? `TOKEN ${String(hint.token).padStart(2, '0')}${hint.layer != null ? ` · LAYER ${String(hint.layer).padStart(2, '0')}` : ''}` : null; return (